template_arfi.txt.template

---
name: ARFI
name_long: All Relevant, Fixed Irrelevant

scripts:
  - get_plot.py
  - merge_descriptives.py
  - merge_metrics.py
  - merge_tds.py

docs:
  - README.md

---

{# This is a template for the ARFI method #}

# version {{ version }}

# Create folder structure. By default, the folder 'output' is used to store output.
mkdir {{ output_folder }}
mkdir {{ output_folder }}/simulation
mkdir {{ output_folder }}/tables
mkdir {{ output_folder }}/tables/metrics
mkdir {{ output_folder }}/tables/time_to_discovery
mkdir {{ output_folder }}/figures
{% for dataset in datasets %}

##################################
### DATASET: {{ dataset.input_file_stem }}
##################################
# Create output folder
mkdir {{ output_folder }}/simulation/{{ dataset.input_file_stem }}/
mkdir {{ output_folder }}/simulation/{{ dataset.input_file_stem }}/metrics

# Collect descriptives about the dataset
mkdir {{ output_folder }}/simulation/{{ dataset.input_file_stem }}/descriptives
python -m asreview data describe {{ dataset.input_file }} -o {{ output_folder }}/simulation/{{ dataset.input_file_stem }}/descriptives/data_stats_{{ dataset.input_file_stem }}.json
{% if not skip_wordclouds %}

# Generate wordcloud visualizations of all datasets
python -m asreview wordcloud {{ dataset.input_file }} -o {{ output_folder }}/figures/wordcloud_{{ dataset.input_file_stem }}.png --width 800 --height 500
python -m asreview wordcloud {{ dataset.input_file }} -o {{ output_folder }}/figures/wordcloud_relevant_{{ dataset.input_file_stem }}.png --width 800 --height 500 --relevant
python -m asreview wordcloud {{ dataset.input_file }} -o {{ output_folder }}/figures/wordcloud_irrelevant_{{ dataset.input_file_stem }}.png --width 800 --height 500 --irrelevant
{% endif %}

# Simulate runs, collect metrics and create plots
mkdir {{ output_folder }}/simulation/{{ dataset.input_file_stem }}/state_files
{% for prior in dataset.priors %}
python -m asreview simulate {{ dataset.input_file }} -s {{ output_folder }}/simulation/{{ dataset.input_file_stem }}/state_files/sim_{{ dataset.input_file_stem }}_{{ prior[0] }}.asreview --prior_record_id {{ " ".join(prior) }} --seed {{ dataset.model_seed }} -m {{ classifier }} -e {{ feature_extractor }} -q {{ query_strategy }} -b {{ balance_strategy }} --n_instances {{ instances_per_query }} --stop_if {{ stop_if }}
python -m asreview metrics {{ output_folder }}/simulation/{{ dataset.input_file_stem }}/state_files/sim_{{ dataset.input_file_stem }}_{{ prior[0] }}.asreview -o {{ output_folder }}/simulation/{{ dataset.input_file_stem }}/metrics/metrics_sim_{{ dataset.input_file_stem }}_{{ prior[0] }}.json
{% endfor %}

# Generate plot and tables for dataset
python {{ scripts_folder }}/get_plot.py -s {{ output_folder }}/simulation/{{ dataset.input_file_stem }}/state_files/ -o {{ output_folder }}/figures/plot_recall_sim_{{ dataset.input_file_stem }}.png --show_legend model
python {{ scripts_folder }}/merge_metrics.py -s {{ output_folder }}/simulation/{{ dataset.input_file_stem }}/metrics/ -o {{ output_folder }}/tables/metrics/metrics_sim_{{ dataset.input_file_stem }}.csv
python {{ scripts_folder }}/merge_tds.py -s {{ output_folder }}/simulation/{{ dataset.input_file_stem }}/metrics/ -o {{ output_folder }}/tables/time_to_discovery/tds_sim_{{ dataset.input_file_stem }}.csv
{% endfor %}

# Merge descriptives and metrics
python {{ scripts_folder }}/merge_descriptives.py
python {{ scripts_folder }}/merge_metrics.py