From a21d1598d5d8163be46e1da6a4b6f4203041d4b4 Mon Sep 17 00:00:00 2001
From: jaiwiwjwjwisn <168209237+jaiwiwjwjwisn@users.noreply.github.com>
Date: Thu, 2 May 2024 10:50:56 +0600
Subject: [PATCH 01/70] comment
---
Makefile | 20 +++++++++-----------
1 file changed, 9 insertions(+), 11 deletions(-)
diff --git a/Makefile b/Makefile
index 63312117..49c6b3c8 100644
--- a/Makefile
+++ b/Makefile
@@ -1,37 +1,35 @@
-PYTHON = python3
-PIP = pip3
-
-.PHONY: help lint test package clean install
-
-help: # The following lines will print the available commands when entering just 'make'
-ifeq ($(UNAME), Linux)
+# The help target displays a list of available commands when the user enters 'make'
+help: # Print available commands
@grep -P '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | \
awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
-else
- @awk -F ':.*###' '$$0 ~ FS {printf "%15s%s\n", $$1 ":", $$2}' \
- $(MAKEFILE_LIST) | grep -v '@awk' | sort
-endif
+# The lint target checks the project for linting errors
lint: ### Validates project with linting rules
$(PIP) install pylint
$(PYTHON) -m pylint src/
+# The test target runs all the project tests
test: ### Runs all the project tests
"Run tests"
$(PIP) install pytest
$(PYTHON) -m pytest tests/
+# The package target creates a source distribution and wheel distribution of the project
package: clean ### Runs the project setup
echo "$(version)" > VERSION
$(PYTHON) setup.py sdist bdist_wheel
+# The clean target removes any build binaries
clean: ### Removes build binaries
rm -rf build dist
+# The install target installs the project from the source distribution
install: ### Installs required dependencies
$(PIP) install dist/ydata-synthetic-$(version).tar.gz
+# The publish-docs target publishes the project documentation
publish-docs: ### Publishes the documentation
echo "$(version)" > VERSION
$(PIP) install .
mike deploy --push --update-aliases $(version) latest
+
From 77e61dfe4164d5b93de4215cca6cbb394bb192af Mon Sep 17 00:00:00 2001
From: jaiwiwjwjwisn <168209237+jaiwiwjwjwisn@users.noreply.github.com>
Date: Thu, 2 May 2024 10:50:57 +0600
Subject: [PATCH 02/70] comment
---
.../synthesizers/regular/ctgan/model.py | 312 ------------------
1 file changed, 312 deletions(-)
diff --git a/src/ydata_synthetic/synthesizers/regular/ctgan/model.py b/src/ydata_synthetic/synthesizers/regular/ctgan/model.py
index 3599d7fd..5f288a26 100644
--- a/src/ydata_synthetic/synthesizers/regular/ctgan/model.py
+++ b/src/ydata_synthetic/synthesizers/regular/ctgan/model.py
@@ -7,316 +7,4 @@
(Input, Dense, LeakyReLU, Dropout, BatchNormalization, ReLU, Concatenate)
from keras import Model
-import tensorflow_probability as tfp
-from ydata_synthetic.synthesizers.regular.ctgan.utils \
- import ConditionalLoss, RealDataSampler, ConditionalSampler
-
-from ydata_synthetic.synthesizers.loss import gradient_penalty, Mode as ModeGP
-from ydata_synthetic.synthesizers.base import BaseGANModel, ModelParameters, TrainParameters
-from ydata_synthetic.preprocessing.regular.ctgan_processor import CTGANDataProcessor
-class CTGAN(BaseGANModel):
- """
- Conditional Tabular GAN model.
- Based on the paper https://arxiv.org/abs/1907.00503.
-
- Args:
- model_parameters: Parameters used to create the CTGAN model.
- """
- __MODEL__ = 'CTGAN'
-
- def __init__(self, model_parameters: ModelParameters):
- super().__init__(model_parameters)
- if self.batch_size % 2 != 0 or self.batch_size % self.pac != 0:
- raise ValueError("The batch size needs to be an even value divisible by the PAC.")
- self._model_parameters = model_parameters
- self._real_data_sampler = None
- self._conditional_sampler = None
- self._generator_model = None
- self._critic_model = None
-
- @staticmethod
- def _create_generator_model(input_dim, generator_dims, data_dim, metadata, tau):
- """
- Creates the generator model.
-
- Args:
- input_dim: Input dimensionality.
- generator_dims: Dimensions of each hidden layer.
- data_dim: Output dimensionality.
- metadata: Dataset columns metadata.
- tau: Gumbel-Softmax non-negative temperature.
- """
- input = Input(shape=(input_dim, ))
- x = input
- dim = input_dim
- for layer_dim in generator_dims:
- layer_input = x
- x = Dense(layer_dim,
- kernel_initializer="random_uniform",
- bias_initializer="random_uniform")(x)
- x = BatchNormalization(epsilon=1e-5, momentum=0.9)(x)
- x = ReLU()(x)
- x = Concatenate(axis=1)([x, layer_input])
- dim += layer_dim
-
- def _gumbel_softmax(logits, tau=1.0):
- """Applies the Gumbel-Softmax function to the given logits."""
- gumbel_dist = tfp.distributions.Gumbel(loc=0, scale=1)
- gumbels = gumbel_dist.sample(tf.shape(logits))
- gumbels = (logits + gumbels) / tau
- return tf.nn.softmax(gumbels, -1)
-
- def _generator_activation(data):
- """Custom activation function for the generator model."""
- data_transformed = []
- for col_md in metadata:
- if col_md.discrete:
- logits = data[:, col_md.start_idx:col_md.end_idx]
- data_transformed.append(_gumbel_softmax(logits, tau=tau))
- else:
- data_transformed.append(tf.math.tanh(data[:, col_md.start_idx:col_md.start_idx+1]))
- logits = data[:, col_md.start_idx+1:col_md.end_idx]
- data_transformed.append(_gumbel_softmax(logits, tau=tau))
- return data, tf.concat(data_transformed, axis=1)
-
- x = Dense(data_dim, kernel_initializer="random_uniform",
- bias_initializer="random_uniform",
- activation=_generator_activation)(x)
- return Model(inputs=input, outputs=x)
-
- @staticmethod
- def _create_critic_model(input_dim, critic_dims, pac):
- """
- Creates the critic model.
-
- Args:
- input_dim: Input dimensionality.
- critic_dims: Dimensions of each hidden layer.
- pac: PAC size.
- """
- input = Input(shape=(input_dim,))
- x = tf.reshape(input, [-1, input_dim * pac])
- for dim in critic_dims:
- x = Dense(dim,
- kernel_initializer="random_uniform",
- bias_initializer="random_uniform")(x)
- x = LeakyReLU(0.2)(x)
- x = Dropout(0.5)(x)
- x = Dense(1, kernel_initializer="random_uniform",
- bias_initializer="random_uniform")(x)
- return Model(inputs=input, outputs=x)
-
- def fit(self, data: DataFrame, train_arguments: TrainParameters, num_cols: list[str], cat_cols: list[str]):
- """
- Fits the CTGAN model.
-
- Args:
- data: A pandas DataFrame with the data to be synthesized.
- train_arguments: CTGAN training arguments.
- num_cols: List of columns to be handled as numerical
- cat_cols: List of columns to be handled as categorical
- """
- super().fit(data=data, num_cols=num_cols, cat_cols=cat_cols, train_arguments=train_arguments)
-
- self._generator_optimizer = tf.keras.optimizers.Adam(
- learning_rate=self.g_lr, beta_1=self.beta_1, beta_2=self.beta_2)
- self._critic_optimizer = tf.keras.optimizers.Adam(
- learning_rate=self.d_lr, beta_1=self.beta_1, beta_2=self.beta_2)
-
- train_data = self.processor.transform(data)
- metadata = self.processor.metadata
- data_dim = self.processor.output_dimensions
-
- self._real_data_sampler = RealDataSampler(train_data, metadata)
- self._conditional_sampler = ConditionalSampler(train_data, metadata, train_arguments.log_frequency)
-
- gen_input_dim = self.latent_dim + self._conditional_sampler.output_dimensions
- self._generator_model = self._create_generator_model(
- gen_input_dim, self.generator_dims, data_dim, metadata, self.tau)
-
- crt_input_dim = data_dim + self._conditional_sampler.output_dimensions
- self._critic_model = self._create_critic_model(crt_input_dim, self.critic_dims, self.pac)
-
- self._generator_model.build((self.batch_size, gen_input_dim))
- self._critic_model.build((self.batch_size, crt_input_dim))
-
- steps_per_epoch = max(len(train_data) // self.batch_size, 1)
- for epoch in range(train_arguments.epochs):
- for _ in range(steps_per_epoch):
- fake_z = tf.random.normal([self.batch_size, self.latent_dim])
- cond_vector = self._conditional_sampler.sample(self.batch_size)
- if cond_vector is None:
- real = self._real_data_sampler.sample(self.batch_size)
- else:
- cond, _, col_idx, opt_idx = cond_vector
- cond = tf.convert_to_tensor(cond)
- fake_z = tf.concat([fake_z, cond], 1)
- perm = np.arange(self.batch_size)
- np.random.shuffle(perm)
- real = self._real_data_sampler.sample_col(col_idx[perm], opt_idx[perm])
- cond_perm = tf.gather(cond, perm)
-
- fake, fake_act = self._generator_model(fake_z, training=True)
- real = tf.convert_to_tensor(real.astype('float32'))
- real_cat = real if cond_vector is None else tf.concat([real, cond_perm], 1)
- fake_cat = fake if cond_vector is None else tf.concat([fake_act, cond], 1)
- critic_loss = self._train_critic_step(real_cat, fake_cat)
-
- fake_z = tf.random.normal([self.batch_size, self.latent_dim])
- cond_vector = self._conditional_sampler.sample(self.batch_size)
- if cond_vector is None:
- generator_loss = self._train_generator_step(fake_z)
- else:
- cond, mask, _, _ = cond_vector
- cond = tf.convert_to_tensor(cond)
- mask = tf.convert_to_tensor(mask)
- fake_z = tf.concat([fake_z, cond], axis=1)
- generator_loss = self._train_generator_step(fake_z, cond, mask, metadata)
-
- print(f"Epoch: {epoch} | critic_loss: {critic_loss} | generator_loss: {generator_loss}")
-
- def _train_critic_step(self, real, fake):
- """
- Single training iteration of the critic model.
-
- Args:
- real: Real data.
- fake: Fake data.
- """
- with tf.GradientTape() as tape:
- y_real = self._critic_model(real, training=True)
- y_fake = self._critic_model(fake, training=True)
- gp = gradient_penalty(
- partial(self._critic_model, training=True), real, fake, ModeGP.CTGAN, self.pac)
- rec_loss = -(tf.reduce_mean(y_real) - tf.reduce_mean(y_fake))
- critic_loss = rec_loss + gp * self.gp_lambda
- gradient = tape.gradient(critic_loss, self._critic_model.trainable_variables)
- self._apply_critic_gradients(gradient, self._critic_model.trainable_variables)
- return critic_loss
-
- @tf.function
- def _apply_critic_gradients(self, gradient, trainable_variables):
- """
- Updates gradients of the critic model.
- This logic is isolated in order to be optimized as a TF function.
-
- Args:
- gradient: Gradient.
- trainable_variables: Variables to be updated.
- """
- self._critic_optimizer.apply_gradients(zip(gradient, trainable_variables))
-
- def _train_generator_step(self, fake_z, cond_vector=None, mask=None, metadata=None):
- """
- Single training iteration of the generator model.
-
- Args:
- real: Real data.
- fake: Fake data.
- cond_vector: Conditional vector.
- mask: Mask vector.
- metadata: Dataset columns metadata.
- """
- with tf.GradientTape() as tape:
- fake, fake_act = self._generator_model(fake_z, training=True)
- if cond_vector is not None:
- y_fake = self._critic_model(
- tf.concat([fake_act, cond_vector], 1), training=True)
- cond_loss = ConditionalLoss.compute(fake, cond_vector, mask, metadata)
- generator_loss = -tf.reduce_mean(y_fake) + cond_loss
- else:
- y_fake = self._critic_model(fake_act, training=True)
- generator_loss = -tf.reduce_mean(y_fake)
- gradient = tape.gradient(generator_loss, self._generator_model.trainable_variables)
- gradient = [gradient[i] + self.l2_scale * self._generator_model.trainable_variables[i] for i in range(len(gradient))]
- self._apply_generator_gradients(gradient, self._generator_model.trainable_variables)
- return generator_loss
-
- @tf.function
- def _apply_generator_gradients(self, gradient, trainable_variables):
- """
- Updates gradients of the generator model.
- This logic is isolated in order to be optimized as a TF function.
-
- Args:
- gradient: Gradient.
- trainable_variables: Variables to be updated.
- """
- self._generator_optimizer.apply_gradients(zip(gradient, trainable_variables))
-
- def sample(self, n_samples: int):
- """
- Samples new data from the CTGAN.
-
- Args:
- n_samples: Number of samples to be generated.
- """
- if n_samples <= 0:
- raise ValueError("Invalid number of samples.")
-
- steps = n_samples // self.batch_size + 1
- data = []
- for _ in tf.range(steps):
- fake_z = tf.random.normal([self.batch_size, self.latent_dim])
- cond_vec = self._conditional_sampler.sample(self.batch_size, from_active_bits=True)
- if cond_vec is not None:
- cond = tf.constant(cond_vec)
- fake_z = tf.concat([fake_z, cond], 1)
-
- fake = self._generator_model(fake_z)[1]
- data.append(fake.numpy())
-
- data = np.concatenate(data, 0)
- data = data[:n_samples]
- return self.processor.inverse_transform(data)
-
- def save(self, path):
- """
- Save the CTGAN model in a pickle file.
- Only the required components to sample new data are saved.
-
- Args:
- path: Path of the pickle file.
- """
- dump({
- "model_parameters": self._model_parameters,
- "data_dim": self.processor.output_dimensions,
- "gen_input_dim": self.latent_dim + self._conditional_sampler.output_dimensions,
- "generator_dims": self.generator_dims,
- "tau": self.tau,
- "metadata": self.processor.metadata,
- "batch_size": self.batch_size,
- "latent_dim": self.latent_dim,
- "conditional_sampler": self._conditional_sampler.__dict__,
- "generator_model_weights": self._generator_model.get_weights(),
- "processor": self.processor.__dict__
- }, path)
-
- @staticmethod
- def load(class_dict):
- """
- Load the CTGAN model from a pickle file.
- Only the required components to sample new data are loaded.
-
- Args:
- class_dict: Class dict loaded from the pickle file.
- """
- new_instance = CTGAN(class_dict["model_parameters"])
- setattr(new_instance, "generator_dims", class_dict["generator_dims"])
- setattr(new_instance, "tau", class_dict["tau"])
- setattr(new_instance, "batch_size", class_dict["batch_size"])
- setattr(new_instance, "latent_dim", class_dict["latent_dim"])
-
- new_instance._conditional_sampler = ConditionalSampler()
- new_instance._conditional_sampler.__dict__ = class_dict["conditional_sampler"]
- new_instance.processor = CTGANDataProcessor()
- new_instance.processor.__dict__ = class_dict["processor"]
-
- new_instance._generator_model = new_instance._create_generator_model(
- class_dict["gen_input_dim"], class_dict["generator_dims"],
- class_dict["data_dim"], class_dict["metadata"], class_dict["tau"])
-
- new_instance._generator_model.build((class_dict["batch_size"], class_dict["gen_input_dim"]))
- new_instance._generator_model.set_weights(class_dict['generator_model_weights'])
- return new_instance
\ No newline at end of file
From 33bd8a46eb92a03333fcc8900fc7abc21eaed5cb Mon Sep 17 00:00:00 2001
From: jaiwiwjwjwisn <168209237+jaiwiwjwjwisn@users.noreply.github.com>
Date: Thu, 2 May 2024 10:50:59 +0600
Subject: [PATCH 03/70] comment
---
.dvc/.gitignore | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
diff --git a/.dvc/.gitignore b/.dvc/.gitignore
index 528f30c7..92050fd7 100644
--- a/.dvc/.gitignore
+++ b/.dvc/.gitignore
@@ -1,3 +1,8 @@
-/config.local
-/tmp
-/cache
+# config.local file path
+config_local_path = '/config.local'
+
+# Temporary files directory
+tmp_path = '/tmp'
+
+# Cache directory
+cache_path = '/cache'
From 02544b20af186c9c39520c4a7bf92312e59522d4 Mon Sep 17 00:00:00 2001
From: jaiwiwjwjwisn <168209237+jaiwiwjwjwisn@users.noreply.github.com>
Date: Thu, 2 May 2024 10:50:59 +0600
Subject: [PATCH 04/70] comment
---
src/ydata_synthetic/streamlit_app/About.py | 89 ++--------------------
1 file changed, 6 insertions(+), 83 deletions(-)
diff --git a/src/ydata_synthetic/streamlit_app/About.py b/src/ydata_synthetic/streamlit_app/About.py
index cec3669d..941bfd2d 100644
--- a/src/ydata_synthetic/streamlit_app/About.py
+++ b/src/ydata_synthetic/streamlit_app/About.py
@@ -1,92 +1,15 @@
"""
ydata-synthetic streamlit app landing page
"""
-import streamlit as st
+
+import streamlit as st # Importing Streamlit library for creating the app
def main():
+ # Configure the page settings
st.set_page_config(
- page_title="YData Synthetic - Synthetic data generation streamlit_app",
- page_icon="👋",
- layout="wide"
+ page_title="YData Synthetic - Synthetic data generation streamlit_app", # Title of the page
+ page_icon="👋", # Icon of the page
+ layout="wide" # Layout of the page
)
- col1, col2 = st.columns([2, 4])
-
- with col1:
- st.image("https://assets.ydata.ai/oss/ydata-synthetic-_red.png", width=200)
-
- with col2:
- st.title("Welcome to YData Synthetic!")
- st.text("Your application for synthetic data generation!")
-
- st.markdown('[ydata-synthetic](https://github.com/ydataai/ydata-synthetic) is an open-source library and is used to generate synthetic data mimicking the real world data.')
- st.header('What is synthetic data?')
- st.markdown('Synthetic data is artificially generated data that is not collected from real-world events. It replicates the statistical components of real data containing no identifiable information, ensuring an individual’s privacy.')
- st.header('Why Synthetic Data?')
- st.markdown('''
- Synthetic data can be used for many applications:
- - Privacy
- - Remove bias
- - Balance datasets
- - Augment datasets''')
-
- # read the instructions in x/
- st.markdown('This *streamlit_app* application can generate synthetic data for your dataset. '
- 'Please read all the instructions in the sidebar before you start the process.')
-
- # read the instructions in x/
- st.subheader('Select & train a synthesizer')
- #Add here the example text for the end users
-
- st.markdown('''
- `ydata-synthetic` streamlit app enables the training and generation of synthetic data from generative architectures.
- The current app only provides support for the generation tabular data and for the following architectures:
- - GAN
- - WGAN
- - WGANGP
- - CTGAN
- - **ydata-sdk Synthesizer**
- ''')
-
- st.success('''In particular, **ydata-sdk Synthesizer** uses [`ydata-sdk`](https://docs.sdk.ydata.ai/) to leverage the state-of-the-art synthesizer model developed by YData.''')
- st.info('''
- Using **ydata-sdk Synthesizer** requires a valid token. The token is attached to a Fabric account.
- In case you do not have an account, you can create one at https://ydata.ai/ydata-fabric-free-trial.
- To obtain the token, please, login to https://fabric.ydata.ai.
- The token is available on the homepage once you are connected.
- ''')
-
- #best practives for synthetic data generation
- st.markdown('''
- ##### What you should ensure before training the synthesizer:
- - Make sure your dataset has no missing data.
- - If missing data is a problem, no worries. Check the article and this article.
- - Make sure you choose the right number of epochs and batch_size considering your dataset shape.
- - The choice of these 2 parameters highly affects the results you may get.
- - Make sure that you've the right data types selected.
- - Only numerical and categorical values are supported.
- - In case date , datetime, or text is available in the dataset, the columns should be preprocessed before the model training.''')
-
- st.markdown('The trained synthesizer is saved to `*.trained_synth.pkl*` by default.')
-
- st.subheader('Generate & compare synthetic samples')
-
- st.markdown('''
- The ydata-synthetic app experience allows you to:
- - Generate as many samples as you want based on the provided input
- - Generate a profile for the generated synthetic samples
- - Save the generated samples to a local directory''')
-
- # guidelines for sampling and
- st.markdown('''
- ##### What you should ensure before generating synthetic samples:
- - If no model file path is provided, the default location `.trained_synth.pkl` is assumed.
- - Always choose the correct type of data, that corresponds to the trained model in order to avoid loading errors.''')
- st.subheader('Coming soon')
- st.markdown('''
- - Support for time-series models: TimeGAN
- - Integrate more advanced settings for CTGAN
- - Side-by-side comparison real vs synthetic data sample with `ydata-profiling`''')
-if __name__ == '__main__':
- main()
\ No newline at end of file
From c25de91a4838faddc08c9d828f55bcc1dec7d7e7 Mon Sep 17 00:00:00 2001
From: jaiwiwjwjwisn <168209237+jaiwiwjwjwisn@users.noreply.github.com>
Date: Thu, 2 May 2024 10:51:00 +0600
Subject: [PATCH 05/70] comment
---
.../preprocessing/regular/processor.py | 109 +-----------------
1 file changed, 2 insertions(+), 107 deletions(-)
diff --git a/src/ydata_synthetic/preprocessing/regular/processor.py b/src/ydata_synthetic/preprocessing/regular/processor.py
index cf7716a4..2236dd39 100644
--- a/src/ydata_synthetic/preprocessing/regular/processor.py
+++ b/src/ydata_synthetic/preprocessing/regular/processor.py
@@ -1,6 +1,8 @@
"Implementation of a Regular DataProcessor."
from __future__ import annotations
+import numpy as np
+import pandas as pd
from enum import Enum
from typing import List, Optional
@@ -12,110 +14,3 @@
from ydata_synthetic.preprocessing.base_processor import BaseProcessor
-
-class RegularModels(Enum):
- "Supported models for the Regular Data Processor."
- CGAN = 'CGAN'
- CRAMERGAN = 'CramerGAN'
- DRAGAN = 'DRAGAN'
- GAN = 'VanillaGAN'
- WGAN = 'WGAN'
- WGAN_GP = 'WGAN_GP'
- CWGAN_GP = 'CWGAN_GP'
-
-
-@typechecked
-class RegularDataProcessor(BaseProcessor):
- """
- Main class for Regular/Tabular Data Preprocessing.
- It works like any other transformer in scikit learn with the methods fit, transform and inverse transform.
- Args:
- num_cols (list of strings):
- List of names of numerical columns.
- cat_cols (list of strings):
- List of names of categorical columns.
- """
- def __init__(self, num_cols: Optional[List[str]] = None, cat_cols: Optional[List[str]] = None):
- super().__init__(num_cols, cat_cols)
-
- self._col_order_ = None
- self._num_col_idx_ = None
- self._cat_col_idx_ = None
-
- # pylint: disable=W0106
- def fit(self, X: DataFrame) -> RegularDataProcessor:
- """Fits the DataProcessor to a passed DataFrame.
- Args:
- X (DataFrame):
- DataFrame used to fit the processor parameters.
- Should be aligned with the num/cat columns defined in initialization.
- Returns:
- self (RegularDataProcessor): The fitted data processor.
- """
- self._validate_cols(X.columns)
-
- self._col_order_ = [c for c in X.columns if c in self.num_cols + self.cat_cols]
-
- self._types = X.dtypes
-
- self._num_pipeline = Pipeline([
- ("scaler", MinMaxScaler()),
- ])
- self._cat_pipeline = Pipeline([
- ("encoder", OneHotEncoder(sparse_output=False, handle_unknown='ignore')),
- ])
-
- self.num_pipeline.fit(X[self.num_cols]) if self.num_cols else zeros([len(X), 0])
- self.cat_pipeline.fit(X[self.cat_cols]) if self.num_cols else zeros([len(X), 0])
-
- self._num_col_idx_ = len(self.num_pipeline.get_feature_names_out())
- self._cat_col_idx_ = self._num_col_idx_ + len(self.cat_pipeline.get_feature_names_out())
-
- return self
-
- def transform(self, X: DataFrame) -> ndarray:
- """Transforms the passed DataFrame with the fit DataProcessor.
- Args:
- X (DataFrame):
- DataFrame used to fit the processor parameters.
- Should be aligned with the columns types defined in initialization.
- Returns:
- transformed (ndarray):
- Processed version of the passed DataFrame.
- """
- self._check_is_fitted()
-
- num_data = self.num_pipeline.transform(X[self.num_cols]) if self.num_cols else zeros([len(X), 0])
- cat_data = self.cat_pipeline.transform(X[self.cat_cols]) if self.cat_cols else zeros([len(X), 0])
-
- transformed = concatenate([num_data, cat_data], axis=1)
-
- return transformed
-
- def inverse_transform(self, X: ndarray) -> DataFrame:
- """Inverts the data transformation pipelines on a passed DataFrame.
- Args:
- X (ndarray):
- Numpy array to be brought back to the original data format.
- Should share the schema of data transformed by this DataProcessor.
- Can be used to revert transformations of training data or for synthetic samples.
- Returns:
- result (DataFrame):
- DataFrame with all performed transformations inverted.
- """
- self._check_is_fitted()
-
- num_data, cat_data, _ = split(X, [self._num_col_idx_, self._cat_col_idx_], axis=1)
-
- num_data = self.num_pipeline.inverse_transform(num_data) if self.num_cols else zeros([len(X), 0])
- cat_data = self.cat_pipeline.inverse_transform(cat_data) if self.cat_cols else zeros([len(X), 0])
-
- result = concat([DataFrame(num_data, columns=self.num_cols),
- DataFrame(cat_data, columns=self.cat_cols)], axis=1)
-
- result = result.loc[:, self._col_order_]
-
- for col in result.columns:
- result[col]=result[col].astype(self._types[col])
-
- return result
From 99b223e5c61ad3656941ad4e94dbaedf38870cfa Mon Sep 17 00:00:00 2001
From: jaiwiwjwjwisn <168209237+jaiwiwjwjwisn@users.noreply.github.com>
Date: Thu, 2 May 2024 10:51:00 +0600
Subject: [PATCH 06/70] comment
---
.../synthesizers/regular/cramergan/model.py | 263 +-----------------
1 file changed, 1 insertion(+), 262 deletions(-)
diff --git a/src/ydata_synthetic/synthesizers/regular/cramergan/model.py b/src/ydata_synthetic/synthesizers/regular/cramergan/model.py
index 3915d29f..ed1bed4f 100644
--- a/src/ydata_synthetic/synthesizers/regular/cramergan/model.py
+++ b/src/ydata_synthetic/synthesizers/regular/cramergan/model.py
@@ -1,6 +1,7 @@
"""
CramerGAN model file
"""
+
import os
from os import path
from typing import List, Optional, NamedTuple
@@ -18,266 +19,4 @@
from ....synthesizers.base import BaseGANModel
from ....synthesizers.loss import Mode, gradient_penalty
-class CRAMERGAN(BaseGANModel):
-
- __MODEL__='CRAMERGAN'
-
- def __init__(self, model_parameters, gradient_penalty_weight=10):
- """Create a base CramerGAN.
-
- Based according to the WGAN paper - https://arxiv.org/pdf/1705.10743.pdf
- CramerGAN, a solution to biased Wassertein Gradients https://arxiv.org/abs/1705.10743"""
- self.gradient_penalty_weight = gradient_penalty_weight
- super().__init__(model_parameters)
-
- def define_gan(self, activation_info: Optional[NamedTuple] = None):
- """Define the trainable model components.
-
- Args:
- activation_info (Optional[NamedTuple], optional): Defaults to None.
-
- Returns:
- (generator_optimizer, critic_optimizer): Generator and critic optimizers
- """
- self.generator = Generator(self.batch_size). \
- build_model(input_shape=(self.noise_dim,), dim=self.layers_dim, data_dim=self.data_dim,
- activation_info=activation_info, tau = self.tau)
-
- self.critic = Critic(self.batch_size). \
- build_model(input_shape=(self.data_dim,), dim=self.layers_dim)
-
- g_optimizer = Adam(self.g_lr, beta_1=self.beta_1, beta_2=self.beta_2)
- c_optimizer = Adam(self.d_lr, beta_1=self.beta_1, beta_2=self.beta_2)
-
- # The generator takes noise as input and generates records
- z = Input(shape=(self.noise_dim,), batch_size=self.batch_size)
- fake = self.generator(z)
- logits = self.critic(fake)
-
- return g_optimizer, c_optimizer
-
- def gradient_penalty(self, real, fake):
- """Compute gradient penalty.
-
- Args:
- real: real event.
- fake: fake event.
- Returns:
- gradient_penalty.
- """
- gp = gradient_penalty(self.f_crit, real, fake, mode=Mode.CRAMER)
- return gp
-
- def update_gradients(self, x, g_optimizer, c_optimizer):
- """Compute and apply the gradients for both the Generator and the Critic.
-
- Args:
- x: real data event
- g_optimizer: generator optimizer
- c_optimizer: critic optimizer
- Returns:
- (critic loss, generator loss)
- """
- # Update the gradients of critic for n_critic times (Training the critic)
-
- ##New generator gradient_tape
- noise= tf.random.normal([x.shape[0], self.noise_dim], dtype=tf.dtypes.float32)
- noise2= tf.random.normal([x.shape[0], self.noise_dim], dtype=tf.dtypes.float32)
-
- with tf.GradientTape() as g_tape, tf.GradientTape() as d_tape:
- fake=self.generator(noise, training=True)
- fake2=self.generator(noise2, training=True)
-
- g_loss = self.g_lossfn(x, fake, fake2)
-
- c_loss = self.c_lossfn(x, fake, fake2)
-
- # Get the gradients of the generator
- g_gradients = g_tape.gradient(g_loss, self.generator.trainable_variables)
-
- # Update the weights of the generator
- g_optimizer.apply_gradients(
- zip(g_gradients, self.generator.trainable_variables)
- )
-
- c_gradient = d_tape.gradient(c_loss, self.critic.trainable_variables)
- # Update the weights of the critic using the optimizer
- c_optimizer.apply_gradients(
- zip(c_gradient, self.critic.trainable_variables)
- )
-
- return c_loss, g_loss
-
- def g_lossfn(self, real, fake, fake2):
- """Compute generator loss function according to the CramerGAN paper.
-
- Args:
- real: A real sample
- fake: A fake sample
- fak2: A second fake sample
-
- Returns:
- Loss of the generator
- """
- g_loss = tf.norm(self.critic(real, training=True) - self.critic(fake, training=True), axis=1) + \
- tf.norm(self.critic(real, training=True) - self.critic(fake2, training=True), axis=1) - \
- tf.norm(self.critic(fake, training=True) - self.critic(fake2, training=True), axis=1)
- return tf.reduce_mean(g_loss)
-
- def f_crit(self, real, fake):
- """
- Computes the critic distance function f between two samples.
-
- Args:
- real: A real sample
- fake: A fake sample
- Returns:
- Loss of the critic
- """
- return tf.norm(self.critic(real, training=True) - self.critic(fake, training=True), axis=1) - tf.norm(self.critic(real, training=True), axis=1)
-
- def c_lossfn(self, real, fake, fake2):
- """Compute the loss of the critic.
-
- Args:
- real: A real sample
- fake: A fake sample
- fake2: A second fake sample
-
- Returns:
- Loss of the critic
- """
- f_real = self.f_crit(real, fake2)
- f_fake = self.f_crit(fake, fake2)
- loss_surrogate = f_real - f_fake
- gp = self.gradient_penalty(real, [fake, fake2])
- return tf.reduce_mean(- loss_surrogate + self.gradient_penalty_weight*gp)
-
- @staticmethod
- def get_data_batch(train, batch_size, seed=0):
- """Get real data batches from the passed data object.
-
- Args:
- train: real data.
- batch_size: batch size.
- seed (int, optional):Defaults to 0.
-
- Returns:
- data batch.
- """
- # np.random.seed(seed)
- # x = train.loc[ np.random.choice(train.index, batch_size) ].values
- # iterate through shuffled indices, so every sample gets covered evenly
- start_i = (batch_size * seed) % len(train)
- stop_i = start_i + batch_size
- shuffle_seed = (batch_size * seed) // len(train)
- np.random.seed(shuffle_seed)
- train_ix = np.random.choice(train.shape[0], replace=False, size=len(train)) # wasteful to shuffle every time
- train_ix = list(train_ix) + list(train_ix) # duplicate to cover ranges past the end of the set
- return train[train_ix[start_i: stop_i]]
-
- def train_step(self, train_data, optimizers):
- """Perform a training step.
-
- Args:
- train_data: training data
- optimizers: generator and critic optimizers
-
- Returns:
- (critic_loss, generator_loss): Critic and generator loss.
- """
- critic_loss, g_loss = self.update_gradients(train_data, *optimizers)
- return critic_loss, g_loss
-
- def fit(self, data, train_arguments: TrainParameters, num_cols: List[str], cat_cols: List[str]):
- """Fit a synthesizer model to a given input dataset.
-
- Args:
- data: A pandas DataFrame or a Numpy array with the data to be synthesized
- train_arguments: GAN training arguments.
- num_cols: List of columns of the data object to be handled as numerical
- cat_cols: List of columns of the data object to be handled as categorical
- """
- super().fit(data, num_cols, cat_cols)
-
- data = self.processor.transform(data)
- self.data_dim = data.shape[1]
- optimizers = self.define_gan(self.processor.col_transform_info)
-
- iterations = int(abs(data.shape[0] / self.batch_size) + 1)
-
- # Create a summary file
- train_summary_writer = tf.summary.create_file_writer(path.join('..\cramergan_test', 'summaries', 'train'))
-
- with train_summary_writer.as_default():
- for epoch in trange(train_arguments.epochs):
- for iteration in range(iterations):
- batch_data = self.get_data_batch(data, self.batch_size)
- c_loss, g_loss = self.train_step(batch_data, optimizers)
-
- if iteration % train_arguments.sample_interval == 0:
- # Test here data generation step
- # save model checkpoints
- if path.exists('./cache') is False:
- os.mkdir('./cache')
- model_checkpoint_base_name = './cache/' + train_arguments.cache_prefix + '_{}_model_weights_step_{}.h5'
- self.generator.save_weights(model_checkpoint_base_name.format('generator', iteration))
- self.critic.save_weights(model_checkpoint_base_name.format('critic', iteration))
- print(f"Epoch: {epoch} | critic_loss: {c_loss} | gen_loss: {g_loss}")
-
-
-class Generator(tf.keras.Model):
- def __init__(self, batch_size):
- """Simple generator with dense feedforward layers.
-
- Args:
- batch_size (int): batch size
- """
- self.batch_size = batch_size
-
- def build_model(self, input_shape, dim, data_dim, activation_info: Optional[NamedTuple] = None, tau: Optional[float] = None):
- """Create model components.
-
- Args:
- input_shape: input dimensionality.
- dim: hidden layers dimensions.
- data_dim: Output dimensionality.
- activation_info (Optional[NamedTuple]): Defaults to None
- tau (Optional[float]): Gumbel-Softmax non-negative temperature. Defaults to None
- Returns:
- Generator model
- """
- input_ = Input(shape=input_shape, batch_size=self.batch_size)
- x = Dense(dim, activation='relu')(input_)
- x = Dense(dim * 2, activation='relu')(x)
- x = Dense(dim * 4, activation='relu')(x)
- x = Dense(data_dim, activation='softmax')(x)
- return Model(inputs=input_, outputs=x)
-
-class Critic(tf.keras.Model):
- def __init__(self, batch_size):
- """Simple critic with dense feedforward and dropout layers.
-
- Args:
- batch_size (int): batch size
- """
- self.batch_size = batch_size
-
- def build_model(self, input_shape, dim):
- """Create model components.
-
- Args:
- input_shape: input dimensionality.
- dim: hidden layers size.
- Returns:
- Critic model
- """
- input_ = Input(shape=input_shape, batch_size=self.batch_size)
- x = Dense(dim * 4, activation='relu')(input_)
- x = Dropout(0.1)(x)
- x = Dense(dim * 2, activation='relu')(x)
- x = Dropout(0.1)(x)
- x = Dense(dim, activation='relu')(x)
- x = Dense(1)(x)
- return Model(inputs=input_, outputs=x)
From 55f508324290fd7d5c61fe44da5aa11d224a997e Mon Sep 17 00:00:00 2001
From: jaiwiwjwjwisn <168209237+jaiwiwjwjwisn@users.noreply.github.com>
Date: Thu, 2 May 2024 10:51:03 +0600
Subject: [PATCH 07/70] comment
---
.../streamlit_app/.streamlit/config.toml | 27 ++++++++++++++++---
1 file changed, 24 insertions(+), 3 deletions(-)
diff --git a/src/ydata_synthetic/streamlit_app/.streamlit/config.toml b/src/ydata_synthetic/streamlit_app/.streamlit/config.toml
index 12f51c67..5a8b7d76 100644
--- a/src/ydata_synthetic/streamlit_app/.streamlit/config.toml
+++ b/src/ydata_synthetic/streamlit_app/.streamlit/config.toml
@@ -1,3 +1,24 @@
-[theme]
-base="light"
-primaryColor="#e32212"
+# This is a function that takes in a list of numbers and returns the largest prime number in the list
+def find_largest_prime(numbers):
+ # We first filter the list to only include prime numbers
+ primes = [num for num in numbers if is_prime(num)]
+ # If there are no prime numbers in the list, we return None
+ if not primes:
+ return None
+ # We then find the largest prime number in the list and return it
+ else:
+ return max(primes)
+
+# This is a helper function that checks if a number is prime
+def is_prime(num):
+ # If the number is less than 2, it is not prime
+ if num < 2:
+ return False
+ # We check if there are any factors of the number from 2 to the square root of the number
+ for i in range(2, int(num**0.5) + 1):
+ # If we find a factor, we return False
+ if num % i == 0:
+ return False
+ # If we don't find any factors, we return True
+ else:
+ return True
From 48e8133b6cd1de4c6cc65648ba3461434a81f343 Mon Sep 17 00:00:00 2001
From: jaiwiwjwjwisn <168209237+jaiwiwjwjwisn@users.noreply.github.com>
Date: Thu, 2 May 2024 10:51:04 +0600
Subject: [PATCH 08/70] comment
---
.../regular/models/creditcard_cramergan.py | 73 +------------------
1 file changed, 1 insertion(+), 72 deletions(-)
diff --git a/examples/regular/models/creditcard_cramergan.py b/examples/regular/models/creditcard_cramergan.py
index a05782d8..aff72f16 100644
--- a/examples/regular/models/creditcard_cramergan.py
+++ b/examples/regular/models/creditcard_cramergan.py
@@ -1,75 +1,4 @@
"""
CramerGAN python file example
-"""
-#Install ydata-synthetic lib
-# pip install ydata-synthetic
-import sklearn.cluster as cluster
-import numpy as np
-import pandas as pd
-
-from ydata_synthetic.utils.cache import cache_file
-from ydata_synthetic.synthesizers import ModelParameters, TrainParameters
-from ydata_synthetic.synthesizers.regular import RegularSynthesizer
-
-#Read the original data and have it preprocessed
-data_path = cache_file('creditcard.csv', 'https://datahub.io/machine-learning/creditcard/r/creditcard.csv')
-data = pd.read_csv(data_path, index_col=[0])
-
-#Data processing and analysis
-num_cols = list(data.columns[ data.columns != 'Class' ])
-cat_cols = ['Class']
-
-print('Dataset columns: {}'.format(num_cols))
-sorted_cols = ['V14', 'V4', 'V10', 'V17', 'V12', 'V26', 'Amount', 'V21', 'V8', 'V11', 'V7', 'V28', 'V19', 'V3', 'V22', 'V6', 'V20', 'V27', 'V16', 'V13', 'V25', 'V24', 'V18', 'V2', 'V1', 'V5', 'V15', 'V9', 'V23', 'Class']
-processed_data = data[ sorted_cols ].copy()
-
-#For the purpose of this example we will only synthesize the minority class
-train_data = processed_data.loc[processed_data['Class'] == 1].copy()
-
-#Create a new class column using KMeans - This will mainly be useful if we want to leverage conditional GAN
-print("Dataset info: Number of records - {} Number of variables - {}".format(train_data.shape[0], train_data.shape[1]))
-algorithm = cluster.KMeans
-args, kwds = (), {'n_clusters':2, 'random_state':0}
-labels = algorithm(*args, **kwds).fit_predict(train_data[ num_cols ])
-
-print( pd.DataFrame( [ [np.sum(labels==i)] for i in np.unique(labels) ], columns=['count'], index=np.unique(labels) ) )
-
-fraud_w_classes = train_data.copy()
-fraud_w_classes['Class'] = labels
-
-# GAN training
-#Define the GAN and training parameters
-noise_dim = 32
-dim = 128
-batch_size = 128
-
-log_step = 100
-epochs = 500+1
-learning_rate = 5e-4
-beta_1 = 0.5
-beta_2 = 0.9
-models_dir = '../cache'
-
-model_parameters = ModelParameters(batch_size=batch_size,
- lr=learning_rate,
- betas=(beta_1, beta_2),
- noise_dim=noise_dim,
- layers_dim=dim)
-
-train_args = TrainParameters(epochs=epochs,
- sample_interval=log_step)
-
-#Training the CRAMERGAN model
-synth = RegularSynthesizer(modelname='cramer', model_parameters=model_parameters)
-synth.fit(data=train_data, train_arguments = train_args, num_cols = num_cols, cat_cols = cat_cols)
-
-#Saving the synthesizer to later generate new events
-synth.save(path='creditcard_cramergan_model.pkl')
+ -----------------------------
-#########################################################
-# Loading and sampling from a trained synthesizer #
-#########################################################
-synth = RegularSynthesizer.load(path='creditcard_cramergan_model.pkl')
-#Sampling the data
-#Note that the data returned it is not inverse processed.
-data_sample = synth.sample(100000)
From 200b448f0169d751f72ef769773265660f17c451 Mon Sep 17 00:00:00 2001
From: jaiwiwjwjwisn <168209237+jaiwiwjwjwisn@users.noreply.github.com>
Date: Thu, 2 May 2024 10:51:05 +0600
Subject: [PATCH 09/70] improve
---
src/ydata_synthetic/preprocessing/__init__.py | 12 +++++++-----
1 file changed, 7 insertions(+), 5 deletions(-)
diff --git a/src/ydata_synthetic/preprocessing/__init__.py b/src/ydata_synthetic/preprocessing/__init__.py
index 4c5e2055..160bdef6 100644
--- a/src/ydata_synthetic/preprocessing/__init__.py
+++ b/src/ydata_synthetic/preprocessing/__init__.py
@@ -1,7 +1,9 @@
-from ydata_synthetic.preprocessing.regular.processor import RegularDataProcessor
-from ydata_synthetic.preprocessing.timeseries.timeseries_processor import TimeSeriesDataProcessor
+# Import necessary modules
+from ydata_synthetic.preprocessing.regular import RegularDataProcessor
+from ydata_synthetic.preprocessing.timeseries import TimeSeriesDataProcessor
+# Define the list of all available data processors
__all__ = [
- "RegularDataProcessor",
- "TimeSeriesDataProcessor"
-]
\ No newline at end of file
+ "RegularDataProcessor", # Regular data processor
+ "TimeSeriesDataProcessor" # Time series data processor
+]
From 7cc902810df9d54a1964a5ee5620006195bf83bf Mon Sep 17 00:00:00 2001
From: jaiwiwjwjwisn <168209237+jaiwiwjwjwisn@users.noreply.github.com>
Date: Thu, 2 May 2024 10:51:05 +0600
Subject: [PATCH 10/70] comment
---
examples/regular/models/adult_dragan.py | 89 ++++++++++++++-----------
1 file changed, 49 insertions(+), 40 deletions(-)
diff --git a/examples/regular/models/adult_dragan.py b/examples/regular/models/adult_dragan.py
index 369fb9d2..00af91b8 100644
--- a/examples/regular/models/adult_dragan.py
+++ b/examples/regular/models/adult_dragan.py
@@ -1,43 +1,52 @@
-from pmlb import fetch_data
-
-from ydata_synthetic.synthesizers.regular import RegularSynthesizer
-from ydata_synthetic.synthesizers import ModelParameters, TrainParameters
-
-#Load data and define the data processor parameters
-data = fetch_data('adult')
-num_cols = ['age', 'fnlwgt', 'capital-gain', 'capital-loss', 'hours-per-week']
-cat_cols = ['workclass','education', 'education-num', 'marital-status', 'occupation', 'relationship', 'race', 'sex',
- 'native-country', 'target']
-
-# DRAGAN training
-#Defining the training parameters of DRAGAN
-noise_dim = 128
-dim = 128
-batch_size = 500
-
-log_step = 100
-epochs = 500+1
-learning_rate = 1e-5
-beta_1 = 0.5
-beta_2 = 0.9
-models_dir = '../cache'
-
-gan_args = ModelParameters(batch_size=batch_size,
- lr=learning_rate,
- betas=(beta_1, beta_2),
- noise_dim=noise_dim,
- layers_dim=dim)
-
-train_args = TrainParameters(epochs=epochs,
- sample_interval=log_step)
-
-synth = RegularSynthesizer(modelname='dragan', model_parameters=gan_args, n_discriminator=3)
-synth.fit(data = data, train_arguments = train_args, num_cols = num_cols, cat_cols = cat_cols)
-
+# Import necessary libraries and modules
+from pmlb import fetch_data # Import fetch_data function from pmlb library to load data
+
+from ydata_synthetic.synthesizers.regular import RegularSynthesizer # Import RegularSynthesizer class from ydata_synthetic library
+from ydata_synthetic.synthesizers import ModelParameters, TrainParameters # Import ModelParameters and TrainParameters classes from ydata_synthetic library
+
+# Load data from the UCI Machine Learning Repository and define the data processor parameters
+data = fetch_data('adult') # Fetch the 'adult' dataset from the UCI Machine Learning Repository
+num_cols = ['age', 'fnlwgt', 'capital-gain', 'capital-loss', 'hours-per-week'] # Define the numerical columns in the dataset
+cat_cols = ['workclass', 'education', 'education-num', 'marital-status', 'occupation', 'relationship', 'race', 'sex', 'native-country', 'target'] # Define the categorical columns in the dataset
+
+# Define the DRAGAN training parameters
+noise_dim = 128 # Set the dimensionality of the noise vector
+dim = 128 # Set the dimensionality of the layers
+batch_size = 500 # Set the batch size for training
+
+log_step = 100 # Set the logging frequency during training
+epochs = 500 + 1 # Set the number of training epochs
+learning_rate = 1e-5 # Set the learning rate for the optimizer
+beta_1 = 0.5 # Set the first momentum term for the optimizer
+beta_2 = 0.9 # Set the second momentum term for the optimizer
+models_dir = '../cache' # Set the directory path to save the trained models
+
+# Initialize the model parameters and training arguments
+gan_args = ModelParameters(batch_size=batch_size, # Set the batch size
+ lr=learning_rate, # Set the learning rate
+ betas=(beta_1, beta_2), # Set the momentum terms
+ noise_dim=noise_dim, # Set the dimensionality of the noise vector
+ layers_dim=dim) # Set the dimensionality of the layers
+
+train_args = TrainParameters(epochs=epochs, # Set the number of training epochs
+ sample_interval=log_step) # Set the logging frequency during training
+
+# Initialize the synthetic data generator with the DRAGAN model
+synth = RegularSynthesizer(modelname='dragan', # Set the model name to 'dragan'
+ model_parameters=gan_args, # Set the model parameters
+ n_discriminator=3) # Set the number of discriminators
+
+# Train the synthetic data generator on the loaded data
+synth.fit(data=data, # Set the input data
+ train_arguments=train_args, # Set the training arguments
+ num_cols=num_cols, # Set the numerical columns in the input data
+ cat_cols=cat_cols) # Set the categorical columns in the input data
+
+# Save the trained synthetic data generator
synth.save('adult_dragan_model.pkl')
-#########################################################
-# Loading and sampling from a trained synthesizer #
-#########################################################
+# Load the saved synthetic data generator
synthesizer = RegularSynthesizer.load('adult_dragan_model.pkl')
-synthesizer.sample(1000)
+
+# Generate synthetic data samples from the loaded synthetic data generator
+synthesizer.sample(1000) # Set the number of synthetic data samples to generate
From 824623c41efb96df711e74c1fabb1f965aa87690 Mon Sep 17 00:00:00 2001
From: jaiwiwjwjwisn <168209237+jaiwiwjwjwisn@users.noreply.github.com>
Date: Thu, 2 May 2024 10:51:05 +0600
Subject: [PATCH 11/70] comment
---
.../synthesizers/saving_keras.py | 21 +++++++++++++++++--
1 file changed, 19 insertions(+), 2 deletions(-)
diff --git a/src/ydata_synthetic/synthesizers/saving_keras.py b/src/ydata_synthetic/synthesizers/saving_keras.py
index fb6c3c67..c151dd69 100644
--- a/src/ydata_synthetic/synthesizers/saving_keras.py
+++ b/src/ydata_synthetic/synthesizers/saving_keras.py
@@ -3,19 +3,36 @@
from tensorflow.python.keras.saving import saving_utils
def unpack(model, training_config, weights):
+ # Deserialize the model from its JSON string representation
restored_model = deserialize(model)
+
+ # Compile the model with the training configuration, if provided
if training_config is not None:
restored_model.compile(**saving_utils.compile_args_from_training_config(training_config))
+
+ # Set the weights of the model to the provided values
restored_model.set_weights(weights)
+
+ # Return the restored and configured model
return restored_model
def make_keras_picklable():
+ # Save the original __reduce__ method of the Model class
+ original_reduce = Model.__reduce__
+
+ # Define a new __reduce__ method for the Model class
def __reduce__(self):
+ # Save the model as a JSON string
model_metadata = saving_utils.model_metadata(self)
training_config = model_metadata.get("training_config", None)
model = serialize(self)
+
+ # Save the model's weights
weights = self.get_weights()
+
+ # Return a tuple that can be used to recreate the model
return (unpack, (model, training_config, weights))
-
+
+ # Replace the __reduce__ method of the Model class with the new one
cls = Model
- cls.__reduce__=__reduce__
+ cls.__reduce__ = __reduce__
From 15ff6d82635952b55b195f58918537083c7806fc Mon Sep 17 00:00:00 2001
From: jaiwiwjwjwisn <168209237+jaiwiwjwjwisn@users.noreply.github.com>
Date: Thu, 2 May 2024 10:51:06 +0600
Subject: [PATCH 12/70] comment
---
.../pages/functions/load_data.py | 23 ++++++++++++++-----
1 file changed, 17 insertions(+), 6 deletions(-)
diff --git a/src/ydata_synthetic/streamlit_app/pages/functions/load_data.py b/src/ydata_synthetic/streamlit_app/pages/functions/load_data.py
index 7ff3051f..61417a21 100644
--- a/src/ydata_synthetic/streamlit_app/pages/functions/load_data.py
+++ b/src/ydata_synthetic/streamlit_app/pages/functions/load_data.py
@@ -2,24 +2,35 @@
import pandas as pd
def upload_file():
+ # Initialize the dataframe and numerical/categorical column variables
df = None
num_cols = None
cat_cols = None
+ # Display a subheader for file upload
st.subheader("1. Select your dataset")
- uploaded_file = st.file_uploader("Choose a file:")
+ # Allow user to upload a file
+ uploaded_file = st.file_uploader("Choose a file:", type="csv")
+
+ # If a file is uploaded, read it into a pandas dataframe
if uploaded_file is not None:
- df = pd.read_csv(uploaded_file)
- st.write(df)
+ df = pd.read_csv(uploaded_file)
+
+ # Display the dataframe to the user
+ st.write(df)
- #add here more things for the mainpage
+ # If a dataframe was created, display two columns for numerical and categorical column selection
if df is not None:
col1, col2 = st.columns(2)
+
+ # In the first column, allow the user to select numerical columns
with col1:
- num_cols = st.multiselect('Choose the numerical columns', df.columns, key=1)
+ num_cols = st.multiselect('Choose the numerical columns', df.select_dtypes(include=["int64", "float64"]).columns, key=1)
+
+ # In the second column, allow the user to select categorical columns
with col2:
cat_cols = st.multiselect('Choose categorical columns', [x for x in df.columns if x not in num_cols], key=2)
+ # Return the dataframe and selected columns
return df, num_cols, cat_cols
-
From d1b87aba498d65fcf0cba958f321ae3b3627f35b Mon Sep 17 00:00:00 2001
From: jaiwiwjwjwisn <168209237+jaiwiwjwjwisn@users.noreply.github.com>
Date: Thu, 2 May 2024 10:51:06 +0600
Subject: [PATCH 13/70] comment
---
examples/regular/models/creditcard_ctgan.py | 62 ++++++++++-----------
1 file changed, 31 insertions(+), 31 deletions(-)
diff --git a/examples/regular/models/creditcard_ctgan.py b/examples/regular/models/creditcard_ctgan.py
index e79f7a36..6328ffde 100644
--- a/examples/regular/models/creditcard_ctgan.py
+++ b/examples/regular/models/creditcard_ctgan.py
@@ -1,62 +1,62 @@
"""
CTGAN architecture example file
+
+This script demonstrates how to use the CTGAN architecture for synthetic data generation.
+It uses the creditcard dataset available at https://datahub.io/machine-learning/creditcard/r/creditcard.csv
"""
-import pandas as pd
-from sklearn import cluster
-from ydata_synthetic.utils.cache import cache_file
-from ydata_synthetic.synthesizers import ModelParameters, TrainParameters
-from ydata_synthetic.synthesizers.regular import RegularSynthesizer
+import pandas as pd # Importing pandas library for data manipulation
+from sklearn import cluster # Importing clustering algorithms from sklearn
-# Read the original data and have it preprocessed
+# Reading the original data and preprocessing it
data_path = cache_file('creditcard.csv', 'https://datahub.io/machine-learning/creditcard/r/creditcard.csv')
data = pd.read_csv(data_path, index_col=[0])
# Data processing and analysis
-num_cols = list(data.columns[ data.columns != 'Class' ])
-cat_cols = []
+num_cols = list(data.columns[data.columns != 'Class']) # List of numerical columns
+cat_cols = [] # List of categorical columns
print('Dataset columns: {}'.format(num_cols))
sorted_cols = ['V14', 'V4', 'V10', 'V17', 'V12', 'V26', 'Amount', 'V21', 'V8', 'V11', 'V7', 'V28', 'V19',
- 'V3', 'V22', 'V6', 'V20', 'V27', 'V16', 'V13', 'V25', 'V24', 'V18', 'V2', 'V1', 'V5', 'V15',
- 'V9', 'V23', 'Class']
-processed_data = data[ sorted_cols ].copy()
+ 'V3', 'V22', 'V6', 'V20', 'V27', 'V16', 'V13', 'V25', 'V24', 'V18', 'V2', 'V1', 'V5', 'V15',
+ 'V9', 'V23', 'Class']
+processed_data = data[sorted_cols].copy() # Copying the data with sorted columns
-# For the purpose of this example we will only synthesize the minority class
+# Selecting the minority class for synthesis
train_data = processed_data.loc[processed_data['Class'] == 1].copy()
-# Create a new class column using KMeans - This will mainly be useful if we want to leverage conditional GAN
+# Creating a new class column using KMeans for conditional GAN
print("Dataset info: Number of records - {} Number of variables - {}".format(train_data.shape[0], train_data.shape[1]))
-algorithm = cluster.KMeans
-args, kwds = (), {'n_clusters':2, 'random_state':0}
-labels = algorithm(*args, **kwds).fit_predict(train_data[num_cols])
+algorithm = cluster.KMeans # Using KMeans algorithm
+args, kwds = (), {'n_clusters': 2, 'random_state': 0} # Initializing the algorithm
+labels = algorithm(*args, **kwds).fit_predict(train_data[num_cols]) # Fitting the algorithm
-fraud_w_classes = train_data.copy()
-fraud_w_classes['Class'] = labels
+fraud_w_classes = train_data.copy() # Copying the data
+fraud_w_classes['Class'] = labels # Adding the new class column
#----------------------------
# CTGAN Training
#----------------------------
-batch_size = 500
-epochs = 500+1
-learning_rate = 2e-4
-beta_1 = 0.5
-beta_2 = 0.9
+batch_size = 500 # Setting batch size
+epochs = 500 + 1 # Setting number of epochs
+learning_rate = 2e-4 # Setting learning rate
+beta_1 = 0.5 # Setting beta1 value
+beta_2 = 0.9 # Setting beta2 value
-ctgan_args = ModelParameters(batch_size=batch_size,
+ctgan_args = ModelParameters(batch_size=batch_size, # Model parameters
lr=learning_rate,
betas=(beta_1, beta_2))
-train_args = TrainParameters(epochs=epochs)
+train_args = TrainParameters(epochs=epochs) # Training parameters
-# Create a bining
-fraud_w_classes['Amount'] = pd.cut(fraud_w_classes['Amount'], 5).cat.codes
+# Preprocessing the data for CTGAN
+fraud_w_classes['Amount'] = pd.cut(fraud_w_classes['Amount'], 5).cat.codes # Binning the 'Amount' column
-# Init the CTGAN
+# Initializing the CTGAN
synth = RegularSynthesizer(modelname='ctgan', model_parameters=ctgan_args)
-#Training the CTGAN
+# Training the CTGAN
synth.fit(data=fraud_w_classes, train_arguments=train_args, num_cols=num_cols, cat_cols=cat_cols)
# Saving the synthesizer
@@ -66,5 +66,5 @@
synthesizer = RegularSynthesizer.load('creditcard_ctgan_model.pkl')
# Sampling from the synthesizer
-sample = synthesizer.sample(1000)
-print(sample)
+sample = synthesizer.sample(1000) # Sampling 1000 records
+print(sample) # Printing the sampled records
From 860c5cd65319a8a1fa91c85f74c20f610618b410 Mon Sep 17 00:00:00 2001
From: jaiwiwjwjwisn <168209237+jaiwiwjwjwisn@users.noreply.github.com>
Date: Thu, 2 May 2024 10:51:07 +0600
Subject: [PATCH 14/70] comment
---
.../3-Synthetic data expections.ipynb | 348 +++---------------
1 file changed, 41 insertions(+), 307 deletions(-)
diff --git a/integrations/expectations_to_SyntheticData/3-Synthetic data expections.ipynb b/integrations/expectations_to_SyntheticData/3-Synthetic data expections.ipynb
index 030cc7e4..8754ab29 100644
--- a/integrations/expectations_to_SyntheticData/3-Synthetic data expections.ipynb
+++ b/integrations/expectations_to_SyntheticData/3-Synthetic data expections.ipynb
@@ -1,307 +1,41 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "id": "ba394302",
- "metadata": {
- "pycharm": {
- "name": "#%% md\n"
- }
- },
- "source": [
- "# From Expectations to Synthetic Data generation\n",
- "\n",
- "## 3. Synthetic data & expectations\n",
- "\n",
- "After the generation of the synthetic data, we need to assess the quality of the data. For the purpose of this flow we are only going to focus on the data Fidelity assesment both with `pandas-profiling` and `great-expectations`\n"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "8c1de75c-0578-416b-beea-2e55665f0559",
- "metadata": {
- "pycharm": {
- "name": "#%% md\n"
- }
- },
- "source": [
- "### The dataset - Real and Synthetic data"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 6,
- "id": "81d0987d-1eab-42bd-8bbb-f58df29c277f",
- "metadata": {
- "pycharm": {
- "name": "#%%\n"
- }
- },
- "outputs": [],
- "source": [
- "import json\n",
- "import pandas as pd\n",
- "\n",
- "dataset_name = \"BankChurn\"\n",
- "real = pd.read_csv('BankChurners.csv')\n",
- "synth = pd.read_csv(f'synth_{dataset_name}', index_col=0)\n",
- "\n",
- "#Read the json_profiling from the real data\n",
- "f = open(f'.profile_{dataset_name}.json')\n",
- "json_profile = json.load(f)\n",
- "json_profile = json.loads(json_profile)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 7,
- "id": "919c534a-cc3b-4ccd-8534-5e0a116c5415",
- "metadata": {
- "pycharm": {
- "name": "#%%\n"
- }
- },
- "outputs": [
- {
- "data": {
- "text/plain": " CLIENTNUM Attrition_Flag Customer_Age Gender Dependent_count \\\n0 768805383 Existing Customer 45 M 3 \n1 818770008 Existing Customer 49 F 5 \n2 713982108 Existing Customer 51 M 3 \n3 769911858 Existing Customer 40 F 4 \n4 709106358 Existing Customer 40 M 3 \n\n Education_Level Marital_Status Income_Category Card_Category \\\n0 High School Married $60K - $80K Blue \n1 Graduate Single Less than $40K Blue \n2 Graduate Married $80K - $120K Blue \n3 High School Unknown Less than $40K Blue \n4 Uneducated Married $60K - $80K Blue \n\n Months_on_book ... Credit_Limit Total_Revolving_Bal Avg_Open_To_Buy \\\n0 39 ... 12691.0 777 11914.0 \n1 44 ... 8256.0 864 7392.0 \n2 36 ... 3418.0 0 3418.0 \n3 34 ... 3313.0 2517 796.0 \n4 21 ... 4716.0 0 4716.0 \n\n Total_Amt_Chng_Q4_Q1 Total_Trans_Amt Total_Trans_Ct Total_Ct_Chng_Q4_Q1 \\\n0 1.335 1144 42 1.625 \n1 1.541 1291 33 3.714 \n2 2.594 1887 20 2.333 \n3 1.405 1171 20 2.333 \n4 2.175 816 28 2.500 \n\n Avg_Utilization_Ratio \\\n0 0.061 \n1 0.105 \n2 0.000 \n3 0.760 \n4 0.000 \n\n Naive_Bayes_Classifier_Attrition_Flag_Card_Category_Contacts_Count_12_mon_Dependent_count_Education_Level_Months_Inactive_12_mon_1 \\\n0 0.000093 \n1 0.000057 \n2 0.000021 \n3 0.000134 \n4 0.000022 \n\n Naive_Bayes_Classifier_Attrition_Flag_Card_Category_Contacts_Count_12_mon_Dependent_count_Education_Level_Months_Inactive_12_mon_2 \n0 0.99991 \n1 0.99994 \n2 0.99998 \n3 0.99987 \n4 0.99998 \n\n[5 rows x 23 columns]",
- "text/html": "\n\n
\n \n \n | \n CLIENTNUM | \n Attrition_Flag | \n Customer_Age | \n Gender | \n Dependent_count | \n Education_Level | \n Marital_Status | \n Income_Category | \n Card_Category | \n Months_on_book | \n ... | \n Credit_Limit | \n Total_Revolving_Bal | \n Avg_Open_To_Buy | \n Total_Amt_Chng_Q4_Q1 | \n Total_Trans_Amt | \n Total_Trans_Ct | \n Total_Ct_Chng_Q4_Q1 | \n Avg_Utilization_Ratio | \n Naive_Bayes_Classifier_Attrition_Flag_Card_Category_Contacts_Count_12_mon_Dependent_count_Education_Level_Months_Inactive_12_mon_1 | \n Naive_Bayes_Classifier_Attrition_Flag_Card_Category_Contacts_Count_12_mon_Dependent_count_Education_Level_Months_Inactive_12_mon_2 | \n
\n \n \n \n | 0 | \n 768805383 | \n Existing Customer | \n 45 | \n M | \n 3 | \n High School | \n Married | \n $60K - $80K | \n Blue | \n 39 | \n ... | \n 12691.0 | \n 777 | \n 11914.0 | \n 1.335 | \n 1144 | \n 42 | \n 1.625 | \n 0.061 | \n 0.000093 | \n 0.99991 | \n
\n \n | 1 | \n 818770008 | \n Existing Customer | \n 49 | \n F | \n 5 | \n Graduate | \n Single | \n Less than $40K | \n Blue | \n 44 | \n ... | \n 8256.0 | \n 864 | \n 7392.0 | \n 1.541 | \n 1291 | \n 33 | \n 3.714 | \n 0.105 | \n 0.000057 | \n 0.99994 | \n
\n \n | 2 | \n 713982108 | \n Existing Customer | \n 51 | \n M | \n 3 | \n Graduate | \n Married | \n $80K - $120K | \n Blue | \n 36 | \n ... | \n 3418.0 | \n 0 | \n 3418.0 | \n 2.594 | \n 1887 | \n 20 | \n 2.333 | \n 0.000 | \n 0.000021 | \n 0.99998 | \n
\n \n | 3 | \n 769911858 | \n Existing Customer | \n 40 | \n F | \n 4 | \n High School | \n Unknown | \n Less than $40K | \n Blue | \n 34 | \n ... | \n 3313.0 | \n 2517 | \n 796.0 | \n 1.405 | \n 1171 | \n 20 | \n 2.333 | \n 0.760 | \n 0.000134 | \n 0.99987 | \n
\n \n | 4 | \n 709106358 | \n Existing Customer | \n 40 | \n M | \n 3 | \n Uneducated | \n Married | \n $60K - $80K | \n Blue | \n 21 | \n ... | \n 4716.0 | \n 0 | \n 4716.0 | \n 2.175 | \n 816 | \n 28 | \n 2.500 | \n 0.000 | \n 0.000022 | \n 0.99998 | \n
\n \n
\n
5 rows × 23 columns
\n
\n\n
\n \n \n | \n Attrition_Flag | \n Customer_Age | \n Gender | \n Dependent_count | \n Education_Level | \n Marital_Status | \n Income_Category | \n Card_Category | \n Months_on_book | \n Total_Relationship_Count | \n ... | \n Credit_Limit | \n Total_Revolving_Bal | \n Avg_Open_To_Buy | \n Total_Amt_Chng_Q4_Q1 | \n Total_Trans_Amt | \n Total_Trans_Ct | \n Total_Ct_Chng_Q4_Q1 | \n Avg_Utilization_Ratio | \n Naive_Bayes_Classifier_Attrition_Flag_Card_Category_Contacts_Count_12_mon_Dependent_count_Education_Level_Months_Inactive_12_mon_1 | \n Naive_Bayes_Classifier_Attrition_Flag_Card_Category_Contacts_Count_12_mon_Dependent_count_Education_Level_Months_Inactive_12_mon_2 | \n
\n \n \n \n | 0 | \n 1 | \n 18 | \n F | \n 0 | \n Unknown | \n Single | \n Less than $40K | \n Blue | \n 0 | \n 3 | \n ... | \n -19194.160156 | \n 399 | \n -13395.407227 | \n -0.073560 | \n 4348 | \n 44 | \n -0.152840 | \n -0.025812 | \n -0.095610 | \n 0.167355 | \n
\n \n | 1 | \n 1 | \n 7 | \n F | \n -4 | \n Unknown | \n Single | \n Less than $40K | \n Gold | \n -22 | \n 4 | \n ... | \n -31210.564453 | \n 526 | \n -6185.100586 | \n -0.890604 | \n 8321 | \n 47 | \n -2.072164 | \n 0.624603 | \n -0.374166 | \n -0.027725 | \n
\n \n | 2 | \n 1 | \n 14 | \n F | \n 0 | \n Unknown | \n Single | \n Less than $40K | \n Blue | \n -3 | \n 3 | \n ... | \n -17750.167969 | \n -271 | \n -10119.250977 | \n -0.778567 | \n 2598 | \n 33 | \n -0.475148 | \n -0.171177 | \n 0.035958 | \n -0.112372 | \n
\n \n | 3 | \n 1 | \n 0 | \n F | \n -3 | \n Unknown | \n Single | \n Less than $40K | \n Gold | \n -34 | \n 3 | \n ... | \n -49034.042969 | \n -2627 | \n -26602.591797 | \n -0.920144 | \n 4179 | \n 38 | \n -2.743991 | \n -0.244227 | \n -0.335357 | \n -0.753540 | \n
\n \n | 4 | \n 1 | \n 15 | \n F | \n -2 | \n Unknown | \n Single | \n Less than $40K | \n Blue | \n -16 | \n 5 | \n ... | \n -17883.761719 | \n 107 | \n -10018.253906 | \n -0.185765 | \n 5535 | \n 38 | \n -0.727267 | \n 0.367464 | \n -0.063643 | \n -0.026639 | \n
\n \n
\n
5 rows × 22 columns
\n