Skip to content

fix: data process and processing schema creation#40

Merged
arjunsridhar12345 merged 3 commits into
devfrom
37-fix-data-process-creation
Jul 21, 2026
Merged

fix: data process and processing schema creation#40
arjunsridhar12345 merged 3 commits into
devfrom
37-fix-data-process-creation

Conversation

@arjunsridhar12345

@arjunsridhar12345 arjunsridhar12345 commented Jul 14, 2026

Copy link
Copy Markdown
Collaborator

Attempts to fix #37. Adds the pipeline name to the data process object so it can be merged later in the metadata manager. Adding the pipeline name to the data process requires the same name to be present in the list of pipelines as well. Thus, that has been added here as well.

Sample output is shown below trying to test this fix:

{
   "object_type": "Processing",
   "describedBy": "https://raw.githubusercontent.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/processing.py",
   "schema_version": "2.3.0",
   "data_processes": [
      {
         "object_type": "Data process",
         "process_type": "Other",
         "name": "Other",
         "stage": "Processing",
         "code": {
            "object_type": "Code",
            "url": "",
            "name": null,
            "version": "",
            "commit_hash": null,
            "container": null,
            "run_script": null,
            "language": null,
            "language_version": null,
            "input_data": null,
            "parameters": {
               "input_source": "//allen/aind/stage/vr-foraging/data/860898/860898_2026-07-09T193353Z/behavior"
            },
            "core_dependency": null
         },
         "experimenters": [
            "AIND Scientific Computing"
         ],
         "pipeline_name": "transform_and_upload_v2",
         "start_date_time": "2026-07-10T01:15:29Z",
         "end_date_time": "2026-07-10T01:15:30Z",
         "output_path": null,
         "output_parameters": null,
         "notes": "Data was copied",
         "resources": null
      },
      {
         "object_type": "Data process",
         "process_type": "Compression",
         "name": "Compression",
         "stage": "Processing",
         "code": {
            "object_type": "Code",
            "url": "ghcr.io/allenneuraldynamics/aind-behavior-video-transformation",
            "name": null,
            "version": "0.2.1",
            "commit_hash": null,
            "container": null,
            "run_script": null,
            "language": null,
            "language_version": null,
            "input_data": null,
            "parameters": {
               "input_source": "//allen/aind/stage/vr-foraging/data/860898/860898_2026-07-09T193353Z/behavior-videos",
               "output_directory": "/allen/aind/stage/svc_aind_airflow/prod/860898_2026-07-09_19-33-53_aa0568f31b/behavior-videos"
            },
            "core_dependency": null
         },
         "experimenters": [
            "AIND Scientific Computing"
         ],
         "pipeline_name": "transform_and_upload_v2",
         "start_date_time": "2026-07-10T01:15:29Z",
         "end_date_time": "2026-07-10T11:36:16Z",
         "output_path": null,
         "output_parameters": null,
         "notes": null,
         "resources": null
      },
      {
         "object_type": "Data process",
         "process_type": "Pipeline",
         "name": "Dynamic foraging NWB packaging",
         "stage": "Processing",
         "code": {
            "object_type": "Code",
            "url": "https://github.com/AllenNeuralDynamics/dynamic-foraging-processing",
            "name": null,
            "version": "0.1.0",
            "commit_hash": null,
            "container": null,
            "run_script": null,
            "language": null,
            "language_version": null,
            "input_data": null,
            "parameters": null,
            "core_dependency": null
         },
         "experimenters": [
            "Alex Piet",
            "Micah Woodard",
            "Bruno Cruz",
            "Arjun Sridhar"
         ],
         "pipeline_name": "dynamic-foraging-processing-pipeline",
         "start_date_time": "2026-07-14T10:28:04.143454-07:00",
         "end_date_time": "2026-07-14T10:28:04.143454-07:00",
         "output_path": null,
         "output_parameters": null,
         "notes": null,
         "resources": null
      }
   ],
   "pipelines": [
      {
         "object_type": "Code",
         "url": "https://github.com/AllenNeuralDynamics/dynamic-foraging-processing",
         "name": "dynamic-foraging-processing-pipeline",
         "version": "0.1.0",
         "commit_hash": null,
         "container": null,
         "run_script": null,
         "language": null,
         "language_version": null,
         "input_data": null,
         "parameters": null,
         "core_dependency": null
      },
      {
         "object_type": "Code",
         "url": "https://github.com/AllenNeuralDynamics/aind-airflow-dags",
         "name": "transform_and_upload_v2",
         "version": "0.39.1",
         "commit_hash": null,
         "container": null,
         "run_script": null,
         "language": "Python",
         "language_version": "3.9.18",
         "input_data": null,
         "parameters": {
            "job_type": "vr_foraging_v2"
         },
         "core_dependency": null
      }
   ],
   "notes": "Processes were reordered by start_date_time",
   "dependency_graph": {
      "Dynamic foraging NWB packaging": [],
      "Other": [],
      "Compression": []
   }
}

@arjunsridhar12345
arjunsridhar12345 marked this pull request as ready for review July 20, 2026 22:14
@arjunsridhar12345
arjunsridhar12345 merged commit c9a286a into dev Jul 21, 2026
3 checks passed
@arjunsridhar12345
arjunsridhar12345 deleted the 37-fix-data-process-creation branch July 21, 2026 18:01
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Labels

None yet

Projects

None yet

Development

Successfully merging this pull request may close these issues.

2 participants