[New Common Data Format] JETS

.github/workflows/fitbot.yml

@@ -84,7 +84,7 @@ jobs:
       run: |
         conda activate nnpdfenv
         cd $RUNFOLDER
-        evolven3fit $RUNCARD $N3FIT_MAXNREP
+        evolven3fit_new evolve $RUNCARD
     # running postfit
     - name: Postfit and upload fit
       shell: bash -l {0}

.github/workflows/tests.yml

@@ -39,6 +39,8 @@ jobs:
       shell: bash -l {0}
       run: |
         echo "$NETRC_FILE" | base64 --decode > ~/.netrc
+        conda install -n base conda-libmamba-solver
+        conda config --set solver libmamba
         conda config --append channels conda-forge
         conda config --prepend channels https://packages.nnpdf.science/public
         conda config --set show_channel_urls true

buildmaster/ATLAS_1JET_8TEV_R06/data.yaml

@@ -0,0 +1,172 @@
+data_central:
+- 36994.0
+- 15405.6
+- 7009.4
+- 3319.54
+- 1627.69
+- 822.52
+- 421.667
+- 223.264
+- 123.88
+- 69.53
+- 40.254
+- 23.062
+- 13.626
+- 8.337
+- 5.1569
+- 3.1134
+- 1.8922
+- 1.1635
+- 0.7111
+- 0.44279
+- 0.27428
+- 0.17043
+- 0.10614
+- 0.06661
+- 0.041322
+- 0.025443
+- 0.015691
+- 0.009639
+- 0.005995
+- 0.003448
+- 0.0014775
+- 0.0003872
+- 5.009e-05
+- 1.42e-06
+- 35232.5
+- 14221.5
+- 6519.5
+- 3066.07
+- 1499.1
+- 757.76
+- 392.639
+- 206.944
+- 114.55
+- 63.287
+- 36.287
+- 20.861
+- 12.404
+- 7.5669
+- 4.5112
+- 2.7158
+- 1.6382
+- 0.9953
+- 0.6045
+- 0.37186
+- 0.22762
+- 0.13906
+- 0.085314
+- 0.051916
+- 0.031895
+- 0.018985
+- 0.011346
+- 0.006725
+- 0.003879
+- 0.0020569
+- 0.0008497
+- 0.0001838
+- 6.58e-06
+- 30557.3
+- 12452.7
+- 5745.3
+- 2630.59
+- 1263.8
+- 638.88
+- 319.543
+- 165.698
+- 90.26
+- 49.844
+- 28.205
+- 15.919
+- 9.281
+- 5.5009
+- 3.2102
+- 1.8731
+- 1.1101
+- 0.6558
+- 0.38074
+- 0.22072
+- 0.12796
+- 0.07427
+- 0.042121
+- 0.023943
+- 0.013329
+- 0.007287
+- 0.003872
+- 0.0019997
+- 0.0010205
+- 0.0004429
+- 0.0001189
+- 5.37e-06
+- 24622.2
+- 9795.9
+- 4208.2
+- 1970.38
+- 949.4
+- 466.875
+- 229.994
+- 116.89
+- 62.17
+- 33.357
+- 18.199
+- 9.772
+- 5.426
+- 3.0905
+- 1.7167
+- 0.90963
+- 0.5003
+- 0.26684
+- 0.13761
+- 0.07125
+- 0.03547
+- 0.017315
+- 0.0082752
+- 0.003744
+- 0.0016581
+- 0.0006909
+- 0.0002583
+- 0.00011
+- 3.58e-05
+- 3.04e-06
+- 20104.9
+- 7668.9
+- 3167.6
+- 1414.54
+- 653.61
+- 311.066
+- 145.509
+- 69.5
+- 34.69
+- 16.754
+- 8.301
+- 4.009
+- 1.952
+- 0.9391
+- 0.4353
+- 0.1905
+- 0.08173
+- 0.032263
+- 0.012586
+- 0.004426
+- 0.001531
+- 0.0004622
+- 0.0001046
+- 1.348e-05
+- 13644.5
+- 5071.1
+- 2024.3
+- 824.82
+- 353.16
+- 143.567
+- 57.724
+- 22.944
+- 9.31
+- 3.705
+- 1.415
+- 0.5147
+- 0.1678
+- 0.04826
+- 0.01333
+- 0.002652
+- 0.000539
+- 4.6e-05

buildmaster/ATLAS_1JET_8TEV_R06/filter.py

@@ -0,0 +1,170 @@
+import yaml
+import numpy as np
+import pandas as pd
+from filter_utils import get_data_values, get_kinematics, fill_df
+
+# ignore pandas warning
+import warnings
+
+warnings.filterwarnings("ignore")
+
+
+def filter_ATLAS_1JET_8TEV_data_kinetic():
+    """
+    write kinematics and central data values
+    in kinematics.yaml and data.yaml files
+    respectively.
+    """
+
+    with open("metadata.yaml", "r") as file:
+        metadata = yaml.safe_load(file)
+
+    version = metadata["hepdata"]["version"]
+    tables = metadata["hepdata"]["tables"]
+
+    # get kinematics from hepdata tables
+    kin = get_kinematics(tables, version)
+
+    # get central values from hepdata tables
+    data_central = get_data_values(tables, version)
+
+    data_central_yaml = {"data_central": data_central}
+    kinematics_yaml = {"bins": kin}
+
+    # write central values and kinematics to yaml file
+    with open("data.yaml", "w") as file:
+        yaml.dump(data_central_yaml, file, sort_keys=False)
+
+    with open("kinematics.yaml", "w") as file:
+        yaml.dump(kinematics_yaml, file, sort_keys=False)
+
+
+def filter_ATLAS_1JET_8TEV_uncertainties(variant='nominal'):
+    """
+    Writes the uncertainties to a .yaml file.
+    Two possible variants are implemented: nominal and decorrelated
+
+    There are three types of uncertainties:
+
+    1. Statistical Uncertainties: ADD, UNCORR
+
+
+    2. Systematic Uncertainties: ADD, CORR
+       Constructed following the exp. prescription:
+
+       - Construct an Error matrix in which
+         each part of an asymmetric error is considered
+         as as separate error (hence dividing by sqrt(2))
+         see also filter_utils/process_error and
+         filter_utils/HEP_table_to_df
+
+
+    3. Luminosity Uncertainty: ATLASLUMI12
+       this uncertainty is correlated with all
+       the other ATLASLUMI12 datasets
+    """
+
+    with open("metadata.yaml", "r") as file:
+        metadata = yaml.safe_load(file)
+
+    version = metadata["hepdata"]["version"]
+    tables = metadata["hepdata"]["tables"]
+
+    # get df of uncertainties
+    dfs = []
+    for table in tables:
+        # uncertainties dataframe
+        df = fill_df(table, version, variant)
+        dfs.append(df)
+
+    df_unc = pd.concat([df for df in dfs], axis=0)
+
+    # statistical errors fully uncorrelated
+    stat_errors = df_unc["stat"].to_numpy()
+
+    # luminosity errors
+    lum_errors = df_unc["syst_lumi"].to_numpy()
+
+    A_corr = df_unc.drop(["stat", "syst_lumi"], axis=1).to_numpy() / np.sqrt(2.0)
+    cov_corr = np.einsum("ij,kj->ik", A_corr, A_corr)
+
+    # error definition
+    error_definition = {
+        f"{col}": {
+            "description": f"correlated systematic {col}",
+            "treatment": "ADD",
+            "type": "CORR",
+        }
+        for col in df_unc.drop(["stat", "syst_lumi"], axis=1).columns
+    }
+
+    error_definition["luminosity_uncertainty"] = {
+        "description": "luminosity uncertainty",
+        "treatment": "ADD",
+        "type": "ATLASLUMI12",
+    }
+
+    error_definition["statistical_uncertainty"] = {
+        "description": "statistical uncertainty",
+        "treatment": "ADD",
+        "type": "UNCORR",
+    }
+
+    # store error in dict
+    error = []
+    for n in range(A_corr.shape[0]):
+        error_value = {}
+        for col, m in zip(df_unc.drop(["stat", "syst_lumi"], axis=1).columns, range(A_corr.shape[1])):
+            error_value[f"{col}"] = float(A_corr[n, m])
+
+        error_value["luminosity_uncertainty"] = float(lum_errors[n])
+        error_value["statistical_uncertainty"] = float(stat_errors[n])
+        error.append(error_value)
+
+    uncertainties_yaml = {"definitions": error_definition, "bins": error}
+
+    # write uncertainties to file
+    if variant=='nominal':
+        with open(f"uncertainties.yaml", "w") as file:
+            yaml.dump(uncertainties_yaml, file, sort_keys=False)
+    else:
+        with open(f"uncertainties_{variant}.yaml", "w") as file:
+            yaml.dump(uncertainties_yaml, file, sort_keys=False)
+
+    # @@@@@@@@@@@ code below for testing only, should be removed at some point @@@@@@@@@@@@#
+    cov_lum = np.einsum("i,j->ij", lum_errors, lum_errors)
+    cov_stat = np.diag(stat_errors**2)
+
+    covmat = cov_corr + cov_stat + cov_lum
+
+    return np.real(covmat)
+
+
+if __name__ == "__main__":
+    # write kinematics and central data values
+    filter_ATLAS_1JET_8TEV_data_kinetic()
+
+    # write uncertainties file
+    filter_ATLAS_1JET_8TEV_uncertainties(variant='nominal')
+
+    # write decorrelated uncertainties file
+    filter_ATLAS_1JET_8TEV_uncertainties(variant='decorrelated')
+
+    ## 
+
+    # # code below for testing only. Should be removed at some point
+    # covmat = filter_ATLAS_1JET_8TEV_uncertainties()
+
+    # from validphys.api import API
+
+    # setname = "ATLAS_1JET_8TEV_R06"
+    # dsinps = [
+    #     {"dataset": setname},
+    # ]
+    # inp = dict(dataset_inputs=dsinps, theoryid=200, use_cuts="internal")
+    # cov = API.dataset_inputs_covmat_from_systematics(**inp)
+
+    # ones = cov / covmat
+    # print(ones)
+    # print(np.max(ones), np.min(ones))
+    # print(np.allclose(ones, np.ones(cov.shape), rtol=1e-5))