dpgen2.superop package

Submodules

dpgen2.superop.block module

class dpgen2.superop.block.ConcurrentLearningBlock(name: str, prep_run_dp_train_op: PrepRunDPTrain, prep_run_explore_op: PrepRunLmp | PrepRunCaly, select_confs_op: Type[OP], prep_run_fp_op: PrepRunFp, collect_data_op: Type[OP], select_confs_config: dict = {'continue_on_failed': False, 'continue_on_num_success': None, 'continue_on_success_ratio': None, 'executor': None, 'parallelism': None, 'template_config': {'envs': None, 'image': 'dptechnology/dpgen2:latest', 'retry_on_transient_error': None, 'timeout': None, 'timeout_as_transient_error': False}}, collect_data_config: dict = {'continue_on_failed': False, 'continue_on_num_success': None, 'continue_on_success_ratio': None, 'executor': None, 'parallelism': None, 'template_config': {'envs': None, 'image': 'dptechnology/dpgen2:latest', 'retry_on_transient_error': None, 'timeout': None, 'timeout_as_transient_error': False}}, upload_python_packages: List[PathLike] | None = None)

Bases: Steps

Attributes:

input_artifacts

input_parameters

keys

output_artifacts

output_parameters

Methods

add(step)

Add a step or a list of parallel steps to the steps

add_slices
convert_to_argo
convert_to_graph
copy
deepcopy
from_dict
from_graph
handle_key
run

property input_artifacts

property input_parameters

property keys

property output_artifacts

property output_parameters

dpgen2.superop.block.make_collect_data_optional_parameter(block_optional_parameter)

dpgen2.superop.block.make_run_dp_train_optional_parameter(block_optional_parameter)

dpgen2.superop.caly_evo_step module

class dpgen2.superop.caly_evo_step.CalyEvoStep(name: str, collect_run_caly: Type[OP], prep_dp_optim: Type[OP], run_dp_optim: Type[OP], prep_config: dict = {'continue_on_failed': False, 'continue_on_num_success': None, 'continue_on_success_ratio': None, 'executor': None, 'parallelism': None, 'template_config': {'envs': None, 'image': 'dptechnology/dpgen2:latest', 'retry_on_transient_error': None, 'timeout': None, 'timeout_as_transient_error': False}}, run_config: dict = {'continue_on_failed': False, 'continue_on_num_success': None, 'continue_on_success_ratio': None, 'executor': None, 'parallelism': None, 'template_config': {'envs': None, 'image': 'dptechnology/dpgen2:latest', 'retry_on_transient_error': None, 'timeout': None, 'timeout_as_transient_error': False}}, upload_python_packages: List[PathLike] | None = None)

Bases: Steps

Attributes:

input_artifacts

input_parameters

keys

output_artifacts

output_parameters

Methods

add(step)

Add a step or a list of parallel steps to the steps

add_slices
convert_to_argo
convert_to_graph
copy
deepcopy
from_dict
from_graph
handle_key
run

property input_artifacts

property input_parameters

property keys

property output_artifacts

property output_parameters

dpgen2.superop.prep_run_calypso module

class dpgen2.superop.prep_run_calypso.PrepRunCaly(name: str, prep_caly_input_op: Type[OP], caly_evo_step_op: OPTemplate, run_caly_model_devi_op: Type[OP], prep_config: dict = {'continue_on_failed': False, 'continue_on_num_success': None, 'continue_on_success_ratio': None, 'executor': None, 'parallelism': None, 'template_config': {'envs': None, 'image': 'dptechnology/dpgen2:latest', 'retry_on_transient_error': None, 'timeout': None, 'timeout_as_transient_error': False}}, run_config: dict = {'continue_on_failed': False, 'continue_on_num_success': None, 'continue_on_success_ratio': None, 'executor': None, 'parallelism': None, 'template_config': {'envs': None, 'image': 'dptechnology/dpgen2:latest', 'retry_on_transient_error': None, 'timeout': None, 'timeout_as_transient_error': False}}, upload_python_packages: List[PathLike] | None = None)

Bases: Steps

Attributes:

input_artifacts

input_parameters

keys

output_artifacts

output_parameters

Methods

add(step)

Add a step or a list of parallel steps to the steps

add_slices
convert_to_argo
convert_to_graph
copy
deepcopy
from_dict
from_graph
handle_key
run

property input_artifacts

property input_parameters

property keys

property output_artifacts

property output_parameters

dpgen2.superop.prep_run_dp_train module

class dpgen2.superop.prep_run_dp_train.ModifyTrainScript(*args, **kwargs)

Bases: OP

Modify the training scripts to prepare them for training tasks in dpgen step.

Read the training scripts modified by finetune, and replace the original template scripts to be compatible with pre-trained models. New templates are returned as op[“template_script”].

Attributes:

key

workflow_name

Methods

execute(ip)	Execute the OP.
get_input_sign()	Get the signature of the inputs
get_output_sign()	Get the signature of the outputs

convert_to_graph
exec_sign_check
from_graph
function
get_info
get_input_artifact_link
get_input_artifact_storage_key
get_opio_info
get_output_artifact_link
get_output_artifact_storage_key
register_output_artifact
superfunction

execute(ip: OPIO) → OPIO

Execute the OP.

Parameters:

ipdict

Input dict with components:

• scripts: (Artifact(Path)) Training scripts from finetune.

• numb_models: (int) Number of DP models to train.

Returns:

opdict

Output dict with components:

• template_script: (List[dict]) One template from one finetuning task. The length of the list should be the same as numb_models.

classmethod get_input_sign()

Get the signature of the inputs

classmethod get_output_sign()

Get the signature of the outputs

class dpgen2.superop.prep_run_dp_train.PrepRunDPTrain(name: str, prep_train_op: ~typing.Type[~dflow.python.op.OP], run_train_op: ~typing.Type[~dpgen2.op.run_dp_train.RunDPTrain], modify_train_script_op: ~typing.Type[~dpgen2.superop.prep_run_dp_train.ModifyTrainScript] = <class 'dpgen2.superop.prep_run_dp_train.ModifyTrainScript'>, prep_config: dict = {'continue_on_failed': False, 'continue_on_num_success': None, 'continue_on_success_ratio': None, 'executor': None, 'parallelism': None, 'template_config': {'envs': None, 'image': 'dptechnology/dpgen2:latest', 'retry_on_transient_error': None, 'timeout': None, 'timeout_as_transient_error': False}}, run_config: dict = {'continue_on_failed': False, 'continue_on_num_success': None, 'continue_on_success_ratio': None, 'executor': None, 'parallelism': None, 'template_config': {'envs': None, 'image': 'dptechnology/dpgen2:latest', 'retry_on_transient_error': None, 'timeout': None, 'timeout_as_transient_error': False}}, upload_python_packages: ~typing.List[~os.PathLike] | None = None, finetune: bool = False, valid_data: ~dflow.common.S3Artifact | None = None)

Bases: Steps

Attributes:

input_artifacts

input_parameters

keys

output_artifacts

output_parameters

Methods

add(step)

Add a step or a list of parallel steps to the steps

add_slices
convert_to_argo
convert_to_graph
copy
deepcopy
from_dict
from_graph
handle_key
run

property input_artifacts

property input_parameters

property keys

property output_artifacts

property output_parameters

dpgen2.superop.prep_run_fp module

class dpgen2.superop.prep_run_fp.PrepRunFp(name: str, prep_op: Type[OP], run_op: Type[OP], prep_config: dict = {'continue_on_failed': False, 'continue_on_num_success': None, 'continue_on_success_ratio': None, 'executor': None, 'parallelism': None, 'template_config': {'envs': None, 'image': 'dptechnology/dpgen2:latest', 'retry_on_transient_error': None, 'timeout': None, 'timeout_as_transient_error': False}}, run_config: dict = {'continue_on_failed': False, 'continue_on_num_success': None, 'continue_on_success_ratio': None, 'executor': None, 'parallelism': None, 'template_config': {'envs': None, 'image': 'dptechnology/dpgen2:latest', 'retry_on_transient_error': None, 'timeout': None, 'timeout_as_transient_error': False}}, upload_python_packages: List[PathLike] | None = None)

Bases: Steps

Attributes:

input_artifacts

input_parameters

keys

output_artifacts

output_parameters

Methods

add(step)

Add a step or a list of parallel steps to the steps

add_slices
convert_to_argo
convert_to_graph
copy
deepcopy
from_dict
from_graph
handle_key
run

property input_artifacts

property input_parameters

property keys

property output_artifacts

property output_parameters

dpgen2.superop.prep_run_lmp module

class dpgen2.superop.prep_run_lmp.PrepRunLmp(name: str, prep_op: Type[OP], run_op: Type[OP], prep_config: dict = {'continue_on_failed': False, 'continue_on_num_success': None, 'continue_on_success_ratio': None, 'executor': None, 'parallelism': None, 'template_config': {'envs': None, 'image': 'dptechnology/dpgen2:latest', 'retry_on_transient_error': None, 'timeout': None, 'timeout_as_transient_error': False}}, run_config: dict = {'continue_on_failed': False, 'continue_on_num_success': None, 'continue_on_success_ratio': None, 'executor': None, 'parallelism': None, 'template_config': {'envs': None, 'image': 'dptechnology/dpgen2:latest', 'retry_on_transient_error': None, 'timeout': None, 'timeout_as_transient_error': False}}, upload_python_packages: List[PathLike] | None = None)

Bases: Steps

Attributes:

input_artifacts

input_parameters

keys

output_artifacts

output_parameters

Methods

add(step)

Add a step or a list of parallel steps to the steps

add_slices
convert_to_argo
convert_to_graph
copy
deepcopy
from_dict
from_graph
handle_key
run

property input_artifacts

property input_parameters

property keys

property output_artifacts

property output_parameters