DRAFT: Add CFAIR Experiemts

experiments/cfair/cfair.yaml

@@ -0,0 +1,94 @@
+---
+__object__: src.explib.base.ExperimentCollection
+name: cfair_ablation
+experiments:
+  - &exp_rad_logN
+    __object__: src.explib.hyperopt.HyperoptExperiment
+    name: cfair_full_radial_logN
+    device: cuda
+    scheduler: &scheduler 
+      __object__: ray.tune.schedulers.ASHAScheduler
+      max_t: 1000000
+      grace_period: 1000000
+      reduction_factor: 2
+    num_hyperopt_samples: &num_hyperopt_samples 4
+    gpus_per_trial: &gpus_per_trial 1
+    cpus_per_trial: &cpus_per_trial 1
+    tuner_params: &tuner_params
+      metric: val_loss
+      mode: min
+    trial_config:
+      logging:
+        images: true
+        "image_shape": [32, 32, 3]
+      dataset: &dataset
+        __object__: src.explib.datasets.Cifar10Split
+        label: 0
+        device: cuda
+      epochs: &epochs 200000
+      patience: &patience 2
+      batch_size: &batch_size 
+        __eval__: tune.choice([32])
+      optim_cfg: &optim 
+        optimizer:
+          __class__: torch.optim.Adam 
+        params:
+          lr: 
+            __eval__: 1e-4
+          weight_decay: 0.0
+
+      model_cfg: 
+        type:
+          __class__: &model src.veriflow.flows.NiceFlow
+        params:
+          soft_training: true
+          training_noise_prior:
+            __object__: pyro.distributions.Uniform
+            low: 
+              __eval__: 1e-20
+            high: 0.01
+          prior_scale: 1.0
+          coupling_layers: 5
+          coupling_nn_layers: [300, 300, 300] 
+          nonlinearity: &nonlinearity 
+            __eval__: tune.choice([torch.nn.ReLU()])
+          split_dim: 392
+          base_distribution: 
+            __object__: src.veriflow.distributions.RadialDistribution       
+            device: cuda
+            p: 1.0
+            loc: 
+              __eval__: torch.zeros(3072).to("cuda")
+            norm_distribution:
+              __object__: pyro.distributions.LogNormal
+              loc: 
+                __eval__: torch.zeros(7).to("cuda")
+              scale: 
+                __eval__: (.5 * torch.ones(1)).to("cuda")
+          use_lu: true
+  - &exp_laplace
+    __overwrites__: *exp_rad_logN
+    name: cfair_full_laplace
+    trial_config:
+      model_cfg: 
+          params:
+            base_distribution:
+              __exact__: 
+                __object__: pyro.distributions.Laplace
+                loc: 
+                  __eval__: torch.zeros(784).to("cuda")
+                scale: 
+                  __eval__: torch.ones(784).to("cuda")
+  - &exp_normal
+    __overwrites__: *exp_rad_logN
+    name: cfair_full_laplace
+    trial_config:
+      model_cfg: 
+          params:
+            base_distribution:
+              __exact__: 
+                __object__: pyro.distributions.Normal
+                loc: 
+                  __eval__: torch.zeros(784).to("cuda")
+                scale: 
+                  __eval__: torch.ones(784).to("cuda")

experiments/cfair/cfair_cpu.yaml

@@ -0,0 +1,66 @@
+---
+__object__: src.explib.base.ExperimentCollection
+name: cfair_ablation
+experiments:
+  - &exp_rad_logN
+    __object__: src.explib.hyperopt.HyperoptExperiment
+    name: cfair_full_radial_logN
+    scheduler: &scheduler 
+      __object__: ray.tune.schedulers.ASHAScheduler
+      max_t: 1000000
+      grace_period: 1000000
+      reduction_factor: 2
+    num_hyperopt_samples: &num_hyperopt_samples 1
+    gpus_per_trial: &gpus_per_trial 0
+    cpus_per_trial: &cpus_per_trial 1
+    tuner_params: &tuner_params
+      metric: val_loss
+      mode: min
+    trial_config:
+      logging:
+        images: true
+        "image_shape": [32, 32, 3]
+      dataset: &dataset
+        __object__: src.explib.datasets.Cifar10Split
+        dataloc: "/home/mustafa/local/dataset/cifar/"
+        label: 0
+      epochs: &epochs 2
+      patience: &patience 1
+      batch_size: &batch_size 
+        __eval__: tune.choice([32])
+      optim_cfg: &optim 
+        optimizer:
+          __class__: torch.optim.Adam 
+        params:
+          lr: 
+            __eval__: 1e-4
+          weight_decay: 0.0
+
+      model_cfg: 
+        type:
+          __class__: &model src.veriflow.flows.NiceFlow
+        params:
+          soft_training: true
+          training_noise_prior:
+            __object__: pyro.distributions.Uniform
+            low: 
+              __eval__: 1e-20
+            high: 0.01
+          prior_scale: 1.0
+          coupling_layers: 10
+          coupling_nn_layers: [300, 300, 300] 
+          nonlinearity: &nonlinearity 
+            __eval__: tune.choice([torch.nn.ReLU()])
+          split_dim: 392
+          base_distribution: 
+            __object__: src.veriflow.distributions.RadialDistribution       
+            device: cpu
+            p:
+              __eval__: math.inf
+            loc: 
+              __eval__: torch.zeros(784).to("cpu")
+            norm_distribution:
+              __object__: pyro.distributions.Exponential
+              rate:
+                __eval__: 1  * torch.ones(1).to("cpu")
+          use_lu: false

experiments/mnist/mnist_0_scaled_14_linf_lognormal_cpu.yaml

@@ -0,0 +1,73 @@
+---
+__object__: src.explib.base.ExperimentCollection
+name: mnist_logNormal_linf_medium_better
+experiments:
+  - &mnist_logNormal_linf_loc_1_scale_05_medium_sized
+    __object__: src.explib.hyperopt.HyperoptExperiment
+    name: mnist_logNormal_linf_loc_1_scale_05_medium_sized
+    scheduler: &scheduler 
+      __object__: ray.tune.schedulers.ASHAScheduler
+      max_t: 1000000
+      grace_period: 1000000
+      reduction_factor: 2
+    num_hyperopt_samples: &num_hyperopt_samples 1
+    gpus_per_trial: &gpus_per_trial 0
+    cpus_per_trial: &cpus_per_trial 1
+    tuner_params: &tuner_params
+      metric: val_loss
+      mode: min
+    device: &device cpu
+    trial_config:
+      logging:
+        images: true
+        "image_shape": [14, 14]
+      dataset: &dataset
+        __object__: src.explib.datasets.MnistSplit
+        scale: true
+        digit: 0
+        device: *device
+        scale_factor: 2
+      epochs: &epochs 2
+      patience: &patience 1
+      batch_size: &batch_size
+        __eval__: tune.choice([16, 32, 64])
+      optim_cfg: &optim
+        optimizer:
+          __class__: torch.optim.Adam
+        params:
+          lr:
+            __eval__: tune.loguniform(1e-7, 1e-4)
+          weight_decay: 0.0
+      model_cfg:
+        type:
+          __class__: &model src.veriflow.flows.NiceFlow
+        params:
+          soft_training: true
+          training_noise_prior:
+            __object__: pyro.distributions.Uniform
+            low:
+              __eval__: 1e-30 * torch.ones(1).to("cpu") #1e-20
+            high:
+                __eval__: 0.001 * torch.ones(1).to("cpu") #0.01
+          prior_scale: 5.0
+          coupling_layers: &coupling_layers
+            __eval__: tune.choice([i for i in range(3, 4)])
+          coupling_nn_layers: &coupling_nn_layers
+            __eval__: "tune.choice([[w] * l for l in [1, 2, 3] for w in [196, 392]])" # tune.choice([[c*32, c*16, c*8, c*16, c*32] for c in [1, 2, 3, 4]] + [[c*64, c*32, c*64]  for c in range(1,5)] + [[c*128] * 2 for c in range(1,5)] + [[c*256] for c in range(1,5)])
+          nonlinearity: &nonlinearity
+            __eval__: tune.choice([torch.nn.ReLU()])
+          split_dim: 50
+          base_distribution:
+            __object__: src.veriflow.distributions.RadialDistribution
+            device: *device
+            p:
+              __eval__: math.inf
+            loc:
+              __eval__: torch.zeros(196).to("cpu")
+            norm_distribution:
+              __object__: pyro.distributions.LogNormal
+              loc:
+                __eval__: (1.2 * torch.ones(1)).to("cpu")
+              scale:
+                __eval__: (0.5 * torch.ones(1)).to("cpu")
+          use_lu: false

experiments/mnist/mnist_0_scaled_14_linf_lognormal_gpu.yaml

@@ -0,0 +1,73 @@
+---
+__object__: src.explib.base.ExperimentCollection
+name: mnist_logNormal_linf_loc_1_scale_05_mnist_14
+experiments:
+  - &mnist_logNormal_linf_loc_1_scale_05_mnist_14
+    __object__: src.explib.hyperopt.HyperoptExperiment
+    name: mnist_logNormal_linf_loc_1_scale_05_medium_sized
+    scheduler: &scheduler 
+      __object__: ray.tune.schedulers.ASHAScheduler
+      max_t: 1000000
+      grace_period: 1000000
+      reduction_factor: 2
+    num_hyperopt_samples: &num_hyperopt_samples 8
+    gpus_per_trial: &gpus_per_trial 2
+    cpus_per_trial: &cpus_per_trial 0
+    tuner_params: &tuner_params
+      metric: val_loss
+      mode: min
+    device: &device cuda
+    trial_config:
+      logging:
+        images: true
+        "image_shape": [14, 14]
+      dataset: &dataset
+        __object__: src.explib.datasets.MnistSplit
+        scale: true
+        digit: 0
+        device: *device
+        scale_factor: 2
+      epochs: &epochs 200000
+      patience: &patience 150
+      batch_size: &batch_size
+        __eval__: tune.choice([16, 32])
+      optim_cfg: &optim
+        optimizer:
+          __class__: torch.optim.Adam
+        params:
+          lr:
+            __eval__: tune.loguniform(1e-4, 1e-2)
+          weight_decay: 0.0
+      model_cfg:
+        type:
+          __class__: &model src.veriflow.flows.NiceFlow
+        params:
+          soft_training: true
+          training_noise_prior:
+            __object__: pyro.distributions.Uniform
+            low:
+              __eval__: 1e-30 * torch.ones(1).to("cuda") #1e-20
+            high:
+                __eval__: 0.001 * torch.ones(1).to("cuda") #0.01
+          prior_scale: 5.0
+          coupling_layers: &coupling_layers
+            __eval__: tune.choice([i for i in range(3, 4)])
+          coupling_nn_layers: &coupling_nn_layers
+            __eval__: "tune.choice([[w] * l for l in [1] for w in [294, 400]])"
+          nonlinearity: &nonlinearity
+            __eval__: tune.choice([torch.nn.ReLU()])
+          split_dim: 98
+          base_distribution:
+            __object__: src.veriflow.distributions.RadialDistribution
+            device: *device
+            p:
+              __eval__: math.inf
+            loc:
+              __eval__: torch.zeros(196).to("cuda")
+            norm_distribution:
+              __object__: pyro.distributions.LogNormal
+              loc:
+                __eval__: torch.ones(1).to("cuda")
+              scale:
+                __eval__: (0.5 * torch.ones(1)).to("cuda")
+          use_lu: false