Adding TabSyn examples and ensemble attack code

.github/workflows/code_checks.yml

@@ -29,17 +29,17 @@ jobs:
   run-code-check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6.0.3
+      - uses: actions/checkout@v7.0.0
 
       - name: Install uv
-        uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39
+        uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3
         with:
           # Install a specific version of uv.
           version: "0.5.21"
           enable-cache: true
 
       - name: "Set up Python"
-        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
         with:
           python-version-file: ".python-version"
 

.github/workflows/docs.yml

@@ -42,7 +42,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v6.0.3
+        uses: actions/checkout@v7.0.0
 
       - name: Install uv
         uses: astral-sh/setup-uv@v8.2.0
@@ -51,7 +51,7 @@ jobs:
           enable-cache: true
 
       - name: Set up Python
-        uses: actions/setup-python@v6.2.0
+        uses: actions/setup-python@v6.3.0
         with:
           python-version-file: ".python-version"
 
@@ -77,7 +77,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v6.0.3
+        uses: actions/checkout@v7.0.0
 
       - name: Configure Git Credentials
         run: |

.github/workflows/integration_tests.yml

@@ -41,17 +41,17 @@ jobs:
   integration-tests:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6.0.3
+      - uses: actions/checkout@v7.0.0
 
       - name: Install uv
-        uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39
+        uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3
         with:
           # Install a specific version of uv.
           version: "0.5.21"
           enable-cache: true
 
       - name: "Set up Python"
-        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
         with:
           python-version-file: ".python-version"
 

.github/workflows/publish.yml

@@ -16,17 +16,17 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install libcurl4-openssl-dev libssl-dev
-      - uses: actions/checkout@v6.0.3
+      - uses: actions/checkout@v7.0.0
 
       - name: Install uv
-        uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39
+        uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3
         with:
           # Install a specific version of uv.
           version: "0.5.21"
           enable-cache: true
 
       - name: "Set up Python"
-        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
         with:
           python-version-file: ".python-version"
 

.github/workflows/unit_tests.yml

@@ -41,17 +41,17 @@ jobs:
   unit-tests:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6.0.3
+      - uses: actions/checkout@v7.0.0
 
       - name: Install uv
-        uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39
+        uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3
         with:
           # Install a specific version of uv.
           version: "0.5.21"
           enable-cache: true
 
       - name: "Set up Python"
-        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
         with:
           python-version-file: ".python-version"
 

.gitignore

@@ -51,6 +51,9 @@ tests/integration/attacks/ensemble/assets/workspace
 tests/integration/assets/tabsyn/processed_data
 tests/integration/assets/tabsyn/results
 
+# Emitted SynthEval analysis config file during metric creation. Unfortunately cannot be turned off...
+SE_analysis_config.json
+
 # Training Logs
 *.err
 *.out

examples/gan/README.md

@@ -94,7 +94,7 @@ Kolmogorov-Smirnov (KS) test, Total Variation Distance (TVD), Correlation Matrix
 and Mutual Information Difference.
 
 To compute those metrics, you can run the command below. The name of the table should be
-defined in the `dataset_meta.json` file, and the file for synthetic data should be under
+defined in the `dataset_meta.json` file, and the data files should be under
 `/data/{table_name}.csv` for the real data and `/results/{table_name}_synthetic.csv`
 for the synthetic data.
 

examples/gan/ensemble_attack/README.md

@@ -60,10 +60,6 @@ python -m examples.gan.ensemble_attack.make_challenge_dataset
 
 ## 4. Training the attack model
 
-> [!NOTE]
-> In the [`config.yaml`](config.yaml) file, the attribute `ensemble_attack.shadow_training.model_name`
-> is what determines this attack will be run with the CTGAN model.
-
 To train the attack models, execute the following command:
 
 ```bash
@@ -81,7 +77,7 @@ To test the attack model against the target model and synthetic data produced on
 [step 2](#2-generating-target-synthetic-data-to-be-tested), please run:
 
 ```bash
-python -m examples.gan.ensemble_attack.test_attack_model
+python -m examples.gan.ensemble_attack.run_test_attack_model
 ```
 
 ## 6. Compute the attack success

examples/gan/ensemble_attack/config.yaml

@@ -36,7 +36,6 @@ ensemble_attack:
     run_metaclassifier_training: true
 
   shadow_training:
-    model_name: ctgan
     model_config:  # Configurations specific for the CTGAN model
       training:
         epochs: 300

examples/gan/ensemble_attack/make_challenge_dataset.py

@@ -22,12 +22,15 @@ def make_challenge_dataset(config: DictConfig) -> None:
         dataset_name = Path(config.training.data_path).stem
         real_data = pd.read_csv(config.training.data_path)
 
+    random_seed = config.ensemble_attack.random_seed
+
     training_data = pd.read_csv(Path(config.results_dir) / f"{dataset_name}_sampled.csv")
     id_column = config.ensemble_attack.table_id_column_name
-    untrained_data = real_data[~real_data[id_column].isin(training_data[id_column])].sample(len(training_data))
+    untrained_data = real_data[~real_data[id_column].isin(training_data[id_column])]
+    sampled_untrained_data = untrained_data.sample(len(training_data), random_state=random_seed)
 
-    challenge_data = pd.concat([training_data, untrained_data])
-    challenge_data_labels = np.concatenate([np.ones(len(training_data)), np.zeros(len(untrained_data))])
+    challenge_data = pd.concat([training_data, sampled_untrained_data])
+    challenge_data_labels = np.concatenate([np.ones(len(training_data)), np.zeros(len(sampled_untrained_data))])
 
     processed_attack_data_path = Path(config.ensemble_attack.data_paths.processed_attack_data_path)
     processed_attack_data_path.mkdir(parents=True, exist_ok=True)
@@ -39,6 +42,8 @@ def make_challenge_dataset(config: DictConfig) -> None:
     log(INFO, f"Saving challenge labels to {challenge_label_path}")
     np.save(challenge_label_path, challenge_data_labels)
 
+    log(INFO, "Done!")
+
 
 if __name__ == "__main__":
     make_challenge_dataset()

examples/gan/ensemble_attack/utils.py

@@ -1,4 +1,5 @@
 import json
+import os
 from pathlib import Path
 from typing import Any
 
@@ -40,6 +41,14 @@ def make_training_config(config: DictConfig) -> dict[Any, Any]:
     Returns:
         The ensemble attack training config for the CTGAN model.
     """
+    base_data_dir = str
+    if "base_data_dir" in config:
+        base_data_dir = config.base_data_dir
+    elif "data_dir" in config:
+        base_data_dir = config.data_dir
+    else:
+        raise ValueError("Either base_data_dir or data_dir must be provided in the config.")
+
     # Saving the model config from the config.yaml into a json file
     # because that's what the ensemble attack code will be looking for
     training_config_path = Path(config.ensemble_attack.shadow_training.training_json_config_paths.training_config_path)
@@ -48,10 +57,10 @@ def make_training_config(config: DictConfig) -> dict[Any, Any]:
         training_config = OmegaConf.to_container(config.ensemble_attack.shadow_training.model_config, resolve=True)
         assert isinstance(training_config, dict), "Training config must be a dictionary."
         training_config["general"] = {
-            "test_data_dir": config.base_data_dir,
+            "test_data_dir": base_data_dir,
             "sample_prefix": "ctgan",
-            "data_dir": config.base_data_dir,
-            "workspace_dir": str(Path(config.base_data_dir) / "shadow_workspace"),
+            "data_dir": base_data_dir,
+            "workspace_dir": os.path.join(base_data_dir, "shadow_workspace"),
             "exp_name": "pre_trained_model",
         }
         json.dump(training_config, f)

examples/midst_evaluation/run_evaluation.py

@@ -53,7 +53,7 @@ def log_metrics(header: str, results: dict[str, float]) -> None:
     """
     log(INFO, f"\n{header}\n{SEPARATOR}\n")
     for metric_name, metric_value in results.items():
-        log(INFO, rf"Metric: {metric_name}\Score: {metric_value}")
+        log(INFO, rf"Metric: {metric_name}\tScore: {metric_value}")
     log(INFO, f"{SEPARATOR}\n")
 
 
@@ -133,7 +133,6 @@ def should_syntheval_preprocess(cfg: DictConfig, for_privacy: bool) -> bool:
         [
             cfg.ks_tv.run,
             cfg.ci_overlap.run,
-            cfg.ks_tv.run,
             cfg.correlation_diff.run,
             cfg.mean_diff.run,
             cfg.f1_score_diff.run,

examples/tabsyn/README.md

@@ -0,0 +1,96 @@
+# TabSyn Single Table Example
+
+This example will go over training a single-table [TabSyn](https://arxiv.org/abs/2310.09656)
+model and synthesizing data afterwards.
+
+
+## Downloading data
+
+First, we need the data. Download it from this
+[Google Drive link](https://drive.google.com/file/d/1HTgfgeL5GXc8uAGfeQirJrUynK7vFeyb/view?usp=drive_link),
+extract the files and place them in a `/data` folder in within this folder
+(`examples/tabsyn`).
+
+> [!NOTE]
+> If you wish to change the data folder, you can do so by editing the `base_data_dir` attribute
+> of the [`config.yaml`](config.yaml) file.
+
+Here is a description of the files that have been extracted:
+- `trans.csv`: The training data. It consists of information about bank transactions and it
+contains 20,000 data points.
+- `trans_info.json`: Metadata about the `trans.csv` data, with information such as which columns are
+numerical and which are categorical, what is the task type, etc.
+
+
+## Kicking off training
+
+To kick off training, simply run the command below from the project's root folder:
+
+```bash
+python -m examples.tabsyn.train
+```
+
+> [!NOTE]
+> For all the commands in this file, you can specify a custom `config.yaml` file
+> by adding the option `--config-path=/path-to-config`. It should point to the folder
+> where the custom `config.yaml` file is located.
+
+## Training results
+
+The result files will be saved inside a `/results` folder within this folder
+(`examples/tabsyn`).
+
+> [!NOTE]
+> If you wish to change the save folder, you can do so by editing the `results_dir` attribute
+> of the [`config.yaml`](config.yaml) file.
+
+In the `/results/trans` folder, there will be a file called `model.pt`,
+which is a pytorch saved model.
+
+
+## Synthesizing data
+
+To synthesize some data with the trained model, run:
+
+```bash
+python -m examples.tabsyn.synthesize
+```
+
+If there is already a trained model in the `/results` folder, it will use that model.
+Otherwise it will train one from scratch. At the end of the script, it will save the
+synthesized data to `/results/trans/synthetic_data/trans_synthetic.csv`.
+
+
+## Evaluating the quality of the synthetic data
+
+### Alpha Precision
+
+To run a round of evaluation with [Alpha Precision](https://arxiv.org/abs/2301.07573)
+metrics on a set of synthetic data, run the `evaluate.py` script:
+
+```bash
+python -m midst_toolkit.evaluation.quality.scripts.midst_alpha_precision_eval \
+  --synthetic_data_path examples/tabsyn/results/trans/synthetic_data/trans_synthetic.csv \
+  --real_data examples/tabsyn/data/trans_sampled.csv \
+  --meta_info_path examples/tabsyn/data/meta_info.json \
+  --save_directory examples/tabsyn/results/
+```
+
+It will save the evaluation results under the `/results/model.txt` file.
+
+### Additional Metrics
+
+The calculation of additional metrics are set up in the `evaluate.py` file. They are the
+Kolmogorov-Smirnov (KS) test, Total Variation Distance (TVD), Correlation Matrix Difference
+and Mutual Information Difference.
+
+To compute those metrics, you can run the command below. The data files should
+be under `/data/{table_name}.csv` for the real data, `/data/{table_name}_sampled.csv`
+for the sampled data used for training, and `/results/{table_name}_synthetic.csv`
+for the synthetic data.
+
+```bash
+python -m examples.tabsyn.evaluate
+```
+
+The results will be saved in the `/results/evaluation.json` file.