diff --git a/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml b/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml index 724f5a0b87ce3..50b2399564e0d 100644 --- a/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml @@ -143,6 +143,7 @@ body: - cohere - common-ai - common-compat + - common-dataquality - common-io - common-messaging - common-sql diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml index 2848323162932..8aeb2695cc787 100644 --- a/.github/boring-cyborg.yml +++ b/.github/boring-cyborg.yml @@ -114,6 +114,9 @@ labelPRBasedOnFilePath: provider:common-compat: - providers/common/compat/** + provider:common-dataquality: + - providers/common/dataquality/** + provider:common-io: - providers/common/io/** diff --git a/airflow-core/docs/extra-packages-ref.rst b/airflow-core/docs/extra-packages-ref.rst index c2bb02ba55dbd..b6617690de233 100644 --- a/airflow-core/docs/extra-packages-ref.rst +++ b/airflow-core/docs/extra-packages-ref.rst @@ -390,51 +390,53 @@ The entries with ``*`` in the ``Preinstalled`` column indicate that those extras pre-installed when Airflow is installed. -+---------------------+-----------------------------------------------------+--------------------------------------+--------------+ -| extra | install command | enables | Preinstalled | -+=====================+=====================================================+======================================+==============+ -| common-ai | ``pip install 'apache-airflow[common-ai]'`` | Common AI Operators and Hooks | | -+---------------------+-----------------------------------------------------+--------------------------------------+--------------+ -| common-compat | ``pip install 'apache-airflow[common-compat]'`` | Compatibility code for old Airflow | | -+---------------------+-----------------------------------------------------+--------------------------------------+--------------+ -| common-io | ``pip install 'apache-airflow[common-io]'`` | Core IO Operators | | -+---------------------+-----------------------------------------------------+--------------------------------------+--------------+ -| common-messaging | ``pip install 'apache-airflow[common-messaging]'`` | Core Messaging Operators | | -+---------------------+-----------------------------------------------------+--------------------------------------+--------------+ -| common-sql | ``pip install 'apache-airflow[common-sql]'`` | Core SQL Operators | * | -+---------------------+-----------------------------------------------------+--------------------------------------+--------------+ -| ftp | ``pip install 'apache-airflow[ftp]'`` | FTP hooks and operators | * | -+---------------------+-----------------------------------------------------+--------------------------------------+--------------+ -| grpc | ``pip install 'apache-airflow[grpc]'`` | Grpc hooks and operators | | -+---------------------+-----------------------------------------------------+--------------------------------------+--------------+ -| http | ``pip install 'apache-airflow[http]'`` | HTTP hooks, operators and sensors | * | -+---------------------+-----------------------------------------------------+--------------------------------------+--------------+ -| imap | ``pip install 'apache-airflow[imap]'`` | IMAP hooks and sensors | * | -+---------------------+-----------------------------------------------------+--------------------------------------+--------------+ -| jdbc | ``pip install 'apache-airflow[jdbc]'`` | JDBC hooks and operators | | -+---------------------+-----------------------------------------------------+--------------------------------------+--------------+ -| keycloak | ``pip install apache-airflow[keycloak]`` | Keycloak hooks and operators | + -+---------------------+-----------------------------------------------------+--------------------------------------+--------------+ -| microsoft-psrp | ``pip install 'apache-airflow[microsoft-psrp]'`` | PSRP hooks and operators | | -+---------------------+-----------------------------------------------------+--------------------------------------+--------------+ -| microsoft-winrm | ``pip install 'apache-airflow[microsoft-winrm]'`` | WinRM hooks and operators | | -+---------------------+-----------------------------------------------------+--------------------------------------+--------------+ -| openlineage | ``pip install 'apache-airflow[openlineage]'`` | Sending OpenLineage events | | -+---------------------+-----------------------------------------------------+--------------------------------------+--------------+ -| opensearch | ``pip install 'apache-airflow[opensearch]'`` | Opensearch hooks and operators | | -+---------------------+-----------------------------------------------------+--------------------------------------+--------------+ -| papermill | ``pip install 'apache-airflow[papermill]'`` | Papermill hooks and operators | | -+---------------------+-----------------------------------------------------+--------------------------------------+--------------+ -| sftp | ``pip install 'apache-airflow[sftp]'`` | SFTP hooks, operators and sensors | | -+---------------------+-----------------------------------------------------+--------------------------------------+--------------+ -| smtp | ``pip install 'apache-airflow[smtp]'`` | SMTP hooks and operators | | -+---------------------+-----------------------------------------------------+--------------------------------------+--------------+ -| sqlite | ``pip install 'apache-airflow[sqlite]'`` | SQLite hooks and operators | * | -+---------------------+-----------------------------------------------------+--------------------------------------+--------------+ -| ssh | ``pip install 'apache-airflow[ssh]'`` | SSH hooks and operators | | -+---------------------+-----------------------------------------------------+--------------------------------------+--------------+ -| informatica | ``pip install 'apache-airflow[informatica]'`` | Informatica hooks and operators | | -+---------------------+-----------------------------------------------------+--------------------------------------+--------------+ ++---------------------+------------------------------------------------------+--------------------------------------+--------------+ +| extra | install command | enables | Preinstalled | ++=====================+======================================================+======================================+==============+ +| common-ai | ``pip install 'apache-airflow[common-ai]'`` | Common AI Operators and Hooks | | ++---------------------+------------------------------------------------------+--------------------------------------+--------------+ +| common-compat | ``pip install 'apache-airflow[common-compat]'`` | Compatibility code for old Airflow | | ++---------------------+------------------------------------------------------+--------------------------------------+--------------+ +| common-dataquality | ``pip install 'apache-airflow[common-dataquality]'`` | Common Data Quality provider | | ++---------------------+------------------------------------------------------+--------------------------------------+--------------+ +| common-io | ``pip install 'apache-airflow[common-io]'`` | Core IO Operators | | ++---------------------+------------------------------------------------------+--------------------------------------+--------------+ +| common-messaging | ``pip install 'apache-airflow[common-messaging]'`` | Core Messaging Operators | | ++---------------------+------------------------------------------------------+--------------------------------------+--------------+ +| common-sql | ``pip install 'apache-airflow[common-sql]'`` | Core SQL Operators | * | ++---------------------+------------------------------------------------------+--------------------------------------+--------------+ +| ftp | ``pip install 'apache-airflow[ftp]'`` | FTP hooks and operators | * | ++---------------------+------------------------------------------------------+--------------------------------------+--------------+ +| grpc | ``pip install 'apache-airflow[grpc]'`` | Grpc hooks and operators | | ++---------------------+------------------------------------------------------+--------------------------------------+--------------+ +| http | ``pip install 'apache-airflow[http]'`` | HTTP hooks, operators and sensors | * | ++---------------------+------------------------------------------------------+--------------------------------------+--------------+ +| imap | ``pip install 'apache-airflow[imap]'`` | IMAP hooks and sensors | * | ++---------------------+------------------------------------------------------+--------------------------------------+--------------+ +| jdbc | ``pip install 'apache-airflow[jdbc]'`` | JDBC hooks and operators | | ++---------------------+------------------------------------------------------+--------------------------------------+--------------+ +| keycloak | ``pip install apache-airflow[keycloak]`` | Keycloak hooks and operators | | ++---------------------+------------------------------------------------------+--------------------------------------+--------------+ +| microsoft-psrp | ``pip install 'apache-airflow[microsoft-psrp]'`` | PSRP hooks and operators | | ++---------------------+------------------------------------------------------+--------------------------------------+--------------+ +| microsoft-winrm | ``pip install 'apache-airflow[microsoft-winrm]'`` | WinRM hooks and operators | | ++---------------------+------------------------------------------------------+--------------------------------------+--------------+ +| openlineage | ``pip install 'apache-airflow[openlineage]'`` | Sending OpenLineage events | | ++---------------------+------------------------------------------------------+--------------------------------------+--------------+ +| opensearch | ``pip install 'apache-airflow[opensearch]'`` | Opensearch hooks and operators | | ++---------------------+------------------------------------------------------+--------------------------------------+--------------+ +| papermill | ``pip install 'apache-airflow[papermill]'`` | Papermill hooks and operators | | ++---------------------+------------------------------------------------------+--------------------------------------+--------------+ +| sftp | ``pip install 'apache-airflow[sftp]'`` | SFTP hooks, operators and sensors | | ++---------------------+------------------------------------------------------+--------------------------------------+--------------+ +| smtp | ``pip install 'apache-airflow[smtp]'`` | SMTP hooks and operators | | ++---------------------+------------------------------------------------------+--------------------------------------+--------------+ +| sqlite | ``pip install 'apache-airflow[sqlite]'`` | SQLite hooks and operators | * | ++---------------------+------------------------------------------------------+--------------------------------------+--------------+ +| ssh | ``pip install 'apache-airflow[ssh]'`` | SSH hooks and operators | | ++---------------------+------------------------------------------------------+--------------------------------------+--------------+ +| informatica | ``pip install 'apache-airflow[informatica]'`` | Informatica hooks and operators | | ++---------------------+------------------------------------------------------+--------------------------------------+--------------+ Group extras ------------ diff --git a/dev/breeze/doc/images/output_build-docs.svg b/dev/breeze/doc/images/output_build-docs.svg index ca5cb1775bcc0..99bc5dbc0de1e 100644 --- a/dev/breeze/doc/images/output_build-docs.svg +++ b/dev/breeze/doc/images/output_build-docs.svg @@ -244,14 +244,14 @@ apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |  -dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github |    -google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc | jenkins |   -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate |  -yandex | ydb | zendesk]...                                                                                             +| cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging | common.sql | databricks |   +datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook |   +ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk +| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa |      +weaviate | yandex | ydb | zendesk]...                                                                                  Build documents. diff --git a/dev/breeze/doc/images/output_build-docs.txt b/dev/breeze/doc/images/output_build-docs.txt index 7e86964622680..75dd9355bcff2 100644 --- a/dev/breeze/doc/images/output_build-docs.txt +++ b/dev/breeze/doc/images/output_build-docs.txt @@ -1 +1 @@ -22cd66d36e80473f4fac0fa311c5e846 +e22969fb5e92a3efafb744881fea5ef8 diff --git a/dev/breeze/doc/images/output_release-management_add-back-references.svg b/dev/breeze/doc/images/output_release-management_add-back-references.svg index 09e78ba88111a..9df4462030484 100644 --- a/dev/breeze/doc/images/output_release-management_add-back-references.svg +++ b/dev/breeze/doc/images/output_release-management_add-back-references.svg @@ -153,14 +153,14 @@ apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |  -dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github |    -google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc | jenkins |   -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate |  -yandex | ydb | zendesk]...                                                                                             +| cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging | common.sql | databricks |   +datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook |   +ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk +| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa |      +weaviate | yandex | ydb | zendesk]...                                                                                  Command to add back references for documentation to make it backward compatible. diff --git a/dev/breeze/doc/images/output_release-management_add-back-references.txt b/dev/breeze/doc/images/output_release-management_add-back-references.txt index 672e05eb6b26f..152f412a472d3 100644 --- a/dev/breeze/doc/images/output_release-management_add-back-references.txt +++ b/dev/breeze/doc/images/output_release-management_add-back-references.txt @@ -1 +1 @@ -2700b8a52b9a04bb3cf5e365e70c2f40 +963d8b3e84cb64aa0d69026638b9fb1b diff --git a/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg b/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg index cef6e4d3b1d14..7e2dae4729d23 100644 --- a/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg +++ b/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg @@ -165,14 +165,14 @@ [OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.cassandra | apache.drill | apache.druid |        apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | -clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        -common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab |   -facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins |    -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    -ydb | zendesk]...                                                                                                      +clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.dataquality | common.io |      +common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch +| exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica |    +jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |     +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |      +yandex | ydb | zendesk]...                                                                                             Classify each provider's unreleased changes with hard-coded, high-confidence rules, flagging ambiguous commits as  'needs_llm' for an agent/skill to assess. Outputs JSON - a deterministic alternative to the random '--non-interactive' diff --git a/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt b/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt index 6e02c700afdf5..c400d2c1da050 100644 --- a/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt +++ b/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt @@ -1 +1 @@ -4cac13b21eee8b732a46c5a15aec7a4b +b4c6b4c23a30b7f750a9a8316abd2e91 diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg index 2f9692be54fa0..78c943d5732d6 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg @@ -154,14 +154,14 @@ [OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.cassandra | apache.drill | apache.druid |        apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | -clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        -common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab |   -facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins |    -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    -ydb | zendesk]...                                                                                                      +clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.dataquality | common.io |      +common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch +| exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica |    +jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |     +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |      +yandex | ydb | zendesk]...                                                                                             Generates content for issue to test the release. diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt index 6afb289b5854f..7c5cc1b0fb397 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt @@ -1 +1 @@ -bd80bf2dd63a27111b8c5745df5dfb86 +ef90cd93a64627b874f63cca845bddbf diff --git a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg index 64efb0af6b65b..4199019d5c56f 100644 --- a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg +++ b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg @@ -173,15 +173,15 @@ │apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | │ │apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | │ │apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes│ -│| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | │ -│datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab | │ -│facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica│ -│| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | │ -│microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | │ -│opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | │ -│redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake │ -│| sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |│ -│yandex | ydb | zendesk)│ +│| cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging | │ +│common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | │ +│elasticsearch | exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http │ +│| imap | influxdb | informatica | jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql │ +│| microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | │ +│openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | │ +│postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | │ +│singularity | slack | smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata│ +│| trino | vertica | vespa | weaviate | yandex | ydb | zendesk)│ │--provider-versionProvider version to generate metadata for. Only used when --provider-id is specified. Limits    │ │running metadata generation to only this version of the provider. (TEXT)│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt index 7c14e04e63151..bf322a8137ed0 100644 --- a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt +++ b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt @@ -1 +1 @@ -5865810f5a4b12c9199267b7dd6459b3 +0e161f1be5f2a5b8c6094dcf7859f1c8 diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg index 27a1d92de9a4b..2a119b59f74a7 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg @@ -192,14 +192,14 @@ [OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.cassandra | apache.drill | apache.druid |        apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | -clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        -common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab |   -facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins |    -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    -ydb | zendesk]...                                                                                                      +clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.dataquality | common.io |      +common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch +| exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica |    +jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |     +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |      +yandex | ydb | zendesk]...                                                                                             Prepare sdist/whl distributions of Airflow Providers. Each provider directory is wiped with `git clean -fdx`  (preserving .venv, .idea, .vscode) before build to keep in-tree generated files out of the artifact. See dev/breeze  diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt index f054c03806e05..9cc84f8ed57aa 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt @@ -1 +1 @@ -a27c1726f5902e5fdb501ecdee226476 +40982121f81ca247791eee99e22d7214 diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg index b9c71f9c81aec..4785a550ef45e 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg @@ -213,14 +213,14 @@ [OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.cassandra | apache.drill | apache.druid |        apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | -clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        -common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab |   -facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins |    -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    -ydb | zendesk]...                                                                                                      +clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.dataquality | common.io |      +common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch +| exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica |    +jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |     +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |      +yandex | ydb | zendesk]...                                                                                             Prepare CHANGELOG, README and COMMITS information for providers. diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt index a54080b6bfbb5..cab1aa0379851 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt @@ -1 +1 @@ -c33a2f6d00a3a8dbec8b56c1c2d88d54 +0d1c20ffd24edc17351c5f132a1cc968 diff --git a/dev/breeze/doc/images/output_release-management_publish-docs.svg b/dev/breeze/doc/images/output_release-management_publish-docs.svg index 9416d6183c582..59c03ad4d408a 100644 --- a/dev/breeze/doc/images/output_release-management_publish-docs.svg +++ b/dev/breeze/doc/images/output_release-management_publish-docs.svg @@ -192,14 +192,14 @@ apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |  -dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github |    -google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc | jenkins |   -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate |  -yandex | ydb | zendesk]...                                                                                             +| cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging | common.sql | databricks |   +datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook |   +ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk +| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa |      +weaviate | yandex | ydb | zendesk]...                                                                                  Command to publish generated documentation to airflow-site diff --git a/dev/breeze/doc/images/output_release-management_publish-docs.txt b/dev/breeze/doc/images/output_release-management_publish-docs.txt index b6433c39bb9be..585d41d6275cb 100644 --- a/dev/breeze/doc/images/output_release-management_publish-docs.txt +++ b/dev/breeze/doc/images/output_release-management_publish-docs.txt @@ -1 +1 @@ -cc4ad6886b5ff0da1d4f819d2c5dc80d +217c089e723c5584da63135bbf011504 diff --git a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg index a9d2e173ef2b8..8e1ee534c9b46 100644 --- a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg +++ b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg @@ -187,15 +187,15 @@ │apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | │ │apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | │ │atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | │ -│common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud | │ -│dingding | discord | docker | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | │ -│github | google | grpc | hashicorp | http | ibm.mq | imap | influxdb | informatica | jdbc | │ -│jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | │ -│mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle │ -│| pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis | salesforce │ -│| samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | │ -│standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex | ydb | │ -│zendesk)│ +│common.compat | common.dataquality | common.io | common.messaging | common.sql | databricks | │ +│datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab | │ +│facebook | ftp | git | github | google | grpc | hashicorp | http | ibm.mq | imap | influxdb | │ +│informatica | jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | │ +│microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | │ +│opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | │ +│redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake │ +│| sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |│ +│yandex | ydb | zendesk)│ │--provider-versionProvider version to generate the requirements for i.e `2.1.0`. `latest` is also a supported     │ │value to account for the most recent version of the provider (TEXT)│ │--force           Force update providers requirements even if they already exist.│ diff --git a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt index bb3262922cec4..ca3930b91a1a2 100644 --- a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt +++ b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt @@ -1 +1 @@ -84f322f2ffa9ded046c2d3678135a8f7 +7cbc3f1d6c0c4f1745702d867f9e7fdc diff --git a/dev/breeze/doc/images/output_workflow-run_publish-docs.svg b/dev/breeze/doc/images/output_workflow-run_publish-docs.svg index dfb0027b3d7ef..13e7e9bf14620 100644 --- a/dev/breeze/doc/images/output_workflow-run_publish-docs.svg +++ b/dev/breeze/doc/images/output_workflow-run_publish-docs.svg @@ -207,14 +207,14 @@ apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |  -dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github |    -google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc | jenkins |   -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate |  -yandex | ydb | zendesk]...                                                                                             +| cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging | common.sql | databricks |   +datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook |   +ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk +| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa |      +weaviate | yandex | ydb | zendesk]...                                                                                  Trigger publish docs to S3 workflow diff --git a/dev/breeze/doc/images/output_workflow-run_publish-docs.txt b/dev/breeze/doc/images/output_workflow-run_publish-docs.txt index 4c6ec79dba124..a86438aee4e27 100644 --- a/dev/breeze/doc/images/output_workflow-run_publish-docs.txt +++ b/dev/breeze/doc/images/output_workflow-run_publish-docs.txt @@ -1 +1 @@ -e784b5234a05d958f84df96cd5d8d562 +cb87f41a19fbc8895cd38aa6351139ae diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 0e8268c8de4cc..95bdf7999a8de 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -843,7 +843,7 @@ def get_airflow_extras(): { "python-version": "3.10", "airflow-version": "2.11.1", - "remove-providers": "anthropic common.messaging edge3 fab git keycloak informatica common.ai opensearch", + "remove-providers": "anthropic common.messaging common.dataquality edge3 fab git keycloak informatica common.ai opensearch", "run-unit-tests": "true", }, { diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 728ab391dc75b..062642515b36b 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -402,6 +402,7 @@ dataprep Dataproc dataproc DataprocDiagnoseClusterOperator +dataquality DataScan dataScans Dataset diff --git a/providers/common/dataquality/.gitignore b/providers/common/dataquality/.gitignore new file mode 100644 index 0000000000000..bff2d7629604d --- /dev/null +++ b/providers/common/dataquality/.gitignore @@ -0,0 +1 @@ +*.iml diff --git a/providers/common/dataquality/LICENSE b/providers/common/dataquality/LICENSE new file mode 100644 index 0000000000000..11069edd79019 --- /dev/null +++ b/providers/common/dataquality/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/providers/common/dataquality/NOTICE b/providers/common/dataquality/NOTICE new file mode 100644 index 0000000000000..a51bd9390d030 --- /dev/null +++ b/providers/common/dataquality/NOTICE @@ -0,0 +1,5 @@ +Apache Airflow +Copyright 2016-2026 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/providers/common/dataquality/README.rst b/providers/common/dataquality/README.rst new file mode 100644 index 0000000000000..baddbee922875 --- /dev/null +++ b/providers/common/dataquality/README.rst @@ -0,0 +1,44 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Package ``apache-airflow-providers-common-dataquality`` + +Release: ``0.1.0`` + +Common Data Quality Provider + +Provider package +---------------- + +This package is for the ``common.dataquality`` provider. +All classes for this package are included in the ``airflow.providers.common.dataquality`` python package. + +Installation +------------ + +You can install this package on top of an existing Airflow installation via +``pip install apache-airflow-providers-common-dataquality``. For the minimum Airflow version supported, +see ``Requirements`` below. + +Requirements +------------ + +================== ================== +PIP package Version required +================== ================== +``apache-airflow`` ``>=3.0.0`` +================== ================== diff --git a/providers/common/dataquality/docs/changelog.rst b/providers/common/dataquality/docs/changelog.rst new file mode 100644 index 0000000000000..9b489866efc4d --- /dev/null +++ b/providers/common/dataquality/docs/changelog.rst @@ -0,0 +1,33 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + .. NOTE TO CONTRIBUTORS: + Please, only add notes to the Changelog just below the "Changelog" header when there + are some breaking changes and you want to add an explanation to the users on how they + are supposed to deal with them. The changelog is updated and maintained semi-automatically + by release manager. + +``apache-airflow-providers-common-dataquality`` + + +Changelog +--------- + +0.1.0 +..... + +Initial version of the provider. diff --git a/providers/common/dataquality/docs/commits.rst b/providers/common/dataquality/docs/commits.rst new file mode 100644 index 0000000000000..1e364ce1c8375 --- /dev/null +++ b/providers/common/dataquality/docs/commits.rst @@ -0,0 +1,35 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + + .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE + `PROVIDER_COMMITS_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY + + .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN! + +Package apache-airflow-providers-common-dataquality +------------------------------------------------------ + +``Common Data Quality Provider`` + + +This is detailed commit list of changes for versions provider package: ``common.dataquality``. +For high-level changelog, see :doc:`package information including changelog `. + +.. airflow-providers-commits:: diff --git a/providers/common/dataquality/docs/conf.py b/providers/common/dataquality/docs/conf.py new file mode 100644 index 0000000000000..222b45280f1cf --- /dev/null +++ b/providers/common/dataquality/docs/conf.py @@ -0,0 +1,27 @@ +# Disable Flake8 because of all the sphinx imports +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Configuration of Providers docs building.""" + +from __future__ import annotations + +import os + +os.environ["AIRFLOW_PACKAGE_NAME"] = "apache-airflow-providers-common-dataquality" + +from docs.provider_conf import * # noqa: F403 diff --git a/providers/common/dataquality/docs/index.rst b/providers/common/dataquality/docs/index.rst new file mode 100644 index 0000000000000..445360c2d4933 --- /dev/null +++ b/providers/common/dataquality/docs/index.rst @@ -0,0 +1,103 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +``apache-airflow-providers-common-dataquality`` +=============================================== + + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Basics + + Home + Changelog + Security + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: References + + Python API <_api/airflow/providers/common/dataquality/index> + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: System tests + + System Tests <_api/tests/system/common/dataquality/index> + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Resources + + PyPI Repository + Installing from sources + +.. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! + + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Commits + + Detailed list of commits + + +apache-airflow-providers-common-dataquality package +------------------------------------------------------ + +Common Data Quality Provider + + +Release: 0.1.0 + +Provider package +---------------- + +This package is for the ``common.dataquality`` provider. +All classes for this package are included in the ``airflow.providers.common.dataquality`` python package. + +Installation +------------ + +You can install this package on top of an existing Airflow installation via +``pip install apache-airflow-providers-common-dataquality``. +For the minimum Airflow version supported, see ``Requirements`` below. + +Requirements +------------ + +The minimum Apache Airflow version supported by this provider distribution is ``3.0.0``. + +================== ================== +PIP package Version required +================== ================== +``apache-airflow`` ``>=3.0.0`` +================== ================== + +Downloading official packages +----------------------------- + +You can download officially released packages and verify their checksums and signatures from the +`Official Apache Download site `_ + +* `The apache-airflow-providers-common-dataquality 0.1.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-common-dataquality 0.1.0 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/common/dataquality/docs/installing-providers-from-sources.rst b/providers/common/dataquality/docs/installing-providers-from-sources.rst new file mode 100644 index 0000000000000..fdbb17d017579 --- /dev/null +++ b/providers/common/dataquality/docs/installing-providers-from-sources.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: /../../../../devel-common/src/sphinx_exts/includes/installing-providers-from-sources.rst diff --git a/providers/common/dataquality/docs/security.rst b/providers/common/dataquality/docs/security.rst new file mode 100644 index 0000000000000..351ff007ebf2f --- /dev/null +++ b/providers/common/dataquality/docs/security.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: /../../../../devel-common/src/sphinx_exts/includes/security.rst diff --git a/providers/common/dataquality/provider.yaml b/providers/common/dataquality/provider.yaml new file mode 100644 index 0000000000000..ae44126145dd5 --- /dev/null +++ b/providers/common/dataquality/provider.yaml @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +--- +package-name: apache-airflow-providers-common-dataquality +name: Common Data Quality +description: | + Common Data Quality Provider + +state: ready +lifecycle: incubation +source-date-epoch: 1751587200 +build-system: flit_core + +# Note that those versions are maintained by release manager - do not update them manually +# with the exception of case where other provider in sources has >= new provider version. +# In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have +# to be done in the same PR +versions: + - 0.1.0 diff --git a/providers/common/dataquality/pyproject.toml b/providers/common/dataquality/pyproject.toml new file mode 100644 index 0000000000000..fd65bbbbd02ef --- /dev/null +++ b/providers/common/dataquality/pyproject.toml @@ -0,0 +1,123 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + +# IF YOU WANT TO MODIFY THIS FILE EXCEPT DEPENDENCIES, YOU SHOULD MODIFY THE TEMPLATE +# `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY +[build-system] +requires = ["flit_core==3.12.0"] +build-backend = "flit_core.buildapi" + +[project] +name = "apache-airflow-providers-common-dataquality" +version = "0.1.0" +description = "Provider package apache-airflow-providers-common-dataquality for Apache Airflow" +readme = "README.rst" +license = "Apache-2.0" +license-files = ['LICENSE', 'NOTICE'] +authors = [ + {name="Apache Software Foundation", email="dev@airflow.apache.org"}, +] +maintainers = [ + {name="Apache Software Foundation", email="dev@airflow.apache.org"}, +] +keywords = [ "airflow-provider", "common.dataquality", "airflow", "integration" ] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Framework :: Apache Airflow", + "Framework :: Apache Airflow :: Provider", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: System :: Monitoring", +] +requires-python = ">=3.10" + +# The dependencies should be modified in place in the generated file. +# Any change in the dependencies is preserved when the file is regenerated +# Make sure to run ``prek update-providers-dependencies --all-files`` +# After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build`` +dependencies = [ + "apache-airflow>=3.0.0", +] + +[dependency-groups] +dev = [ + "apache-airflow", + "apache-airflow-task-sdk", + "apache-airflow-devel-common", + # Additional devel dependencies (do not remove this line and add extra development dependencies) +] + +# To build docs: +# +# uv run --group docs build-docs +# +# To enable auto-refreshing build with server: +# +# uv run --group docs build-docs --autobuild +# +# To see more options: +# +# uv run --group docs build-docs --help +# +docs = [ + "apache-airflow-devel-common[docs]" +] + +[tool.uv.sources] +# These names must match the names as defined in the pyproject.toml of the workspace items, +# *not* the workspace folder paths +apache-airflow = {workspace = true} +apache-airflow-devel-common = {workspace = true} +apache-airflow-task-sdk = {workspace = true} +apache-airflow-providers-common-sql = {workspace = true} +apache-airflow-providers-standard = {workspace = true} + +[project.urls] +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-common-dataquality/0.1.0" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-common-dataquality/0.1.0/changelog.html" +"Bug Tracker" = "https://github.com/apache/airflow/issues" +"Source Code" = "https://github.com/apache/airflow" +"Slack Chat" = "https://s.apache.org/airflow-slack" +"Mastodon" = "https://fosstodon.org/@airflow" +"YouTube" = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/" + +[project.entry-points."apache_airflow_provider"] +provider_info = "airflow.providers.common.dataquality.get_provider_info:get_provider_info" + +[tool.flit.module] +name = "airflow.providers.common.dataquality" + +# Explicit sdist contents so the build does not rely on VCS information +# (flit 4.0 makes --no-use-vcs the default — see https://github.com/pypa/flit/pull/782). +[tool.flit.sdist] +include = [ + "docs/", + "provider.yaml", + "src/airflow/__init__.py", + "src/airflow/providers/__init__.py", + "src/airflow/providers/common/__init__.py", + "tests/", +] diff --git a/providers/common/dataquality/src/airflow/__init__.py b/providers/common/dataquality/src/airflow/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/common/dataquality/src/airflow/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/common/dataquality/src/airflow/providers/__init__.py b/providers/common/dataquality/src/airflow/providers/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/common/dataquality/src/airflow/providers/common/__init__.py b/providers/common/dataquality/src/airflow/providers/common/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/__init__.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/__init__.py new file mode 100644 index 0000000000000..484a92c3fad31 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/__init__.py @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE +# OVERWRITTEN WHEN PREPARING DOCUMENTATION FOR THE PACKAGES. +# +# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE +# `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY +# +from __future__ import annotations + +import packaging.version + +from airflow import __version__ as airflow_version + +__all__ = ["__version__"] + +__version__ = "0.1.0" + +if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( + "3.0.0" +): + raise RuntimeError( + f"The package `apache-airflow-providers-common-dataquality:{__version__}` needs Apache Airflow 3.0.0+" + ) diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/get_provider_info.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/get_provider_info.py new file mode 100644 index 0000000000000..961705e59b515 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/get_provider_info.py @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! +# +# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE +# `get_provider_info_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY + + +def get_provider_info(): + return { + "package-name": "apache-airflow-providers-common-dataquality", + "name": "Common Data Quality", + "description": "Common Data Quality Provider\n", + } diff --git a/providers/common/dataquality/tests/conftest.py b/providers/common/dataquality/tests/conftest.py new file mode 100644 index 0000000000000..f56ccce0a3f69 --- /dev/null +++ b/providers/common/dataquality/tests/conftest.py @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +pytest_plugins = "tests_common.pytest_plugin" diff --git a/providers/common/dataquality/tests/system/__init__.py b/providers/common/dataquality/tests/system/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/common/dataquality/tests/system/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/common/dataquality/tests/system/common/__init__.py b/providers/common/dataquality/tests/system/common/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/common/dataquality/tests/system/common/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/common/dataquality/tests/system/common/dataquality/__init__.py b/providers/common/dataquality/tests/system/common/dataquality/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/common/dataquality/tests/system/common/dataquality/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/common/dataquality/tests/unit/__init__.py b/providers/common/dataquality/tests/unit/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/common/dataquality/tests/unit/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/common/dataquality/tests/unit/common/__init__.py b/providers/common/dataquality/tests/unit/common/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/common/dataquality/tests/unit/common/dataquality/__init__.py b/providers/common/dataquality/tests/unit/common/dataquality/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/common/dataquality/tests/unit/common/dataquality/test_example.py b/providers/common/dataquality/tests/unit/common/dataquality/test_example.py new file mode 100644 index 0000000000000..18182e8a158fb --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/test_example.py @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from airflow.providers.common.dataquality.get_provider_info import get_provider_info + + +def test_get_provider_info(): + provider_info = get_provider_info() + + assert provider_info["package-name"] == "apache-airflow-providers-common-dataquality" + assert provider_info["name"] == "Common Data Quality" diff --git a/pyproject.toml b/pyproject.toml index 7e28ca58b1c1b..2d0c363e03e1c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -191,6 +191,9 @@ apache-airflow = "airflow.__main__:main" "common.compat" = [ "apache-airflow-providers-common-compat>=1.2.1" ] +"common.dataquality" = [ + "apache-airflow-providers-common-dataquality>=0.1.0" +] "common.io" = [ "apache-airflow-providers-common-io>=1.4.2" ] @@ -441,6 +444,7 @@ apache-airflow = "airflow.__main__:main" "apache-airflow-providers-cohere>=1.4.0", "apache-airflow-providers-common-ai>=0.1.0", "apache-airflow-providers-common-compat>=1.2.1", + "apache-airflow-providers-common-dataquality>=0.1.0", "apache-airflow-providers-common-io>=1.4.2", "apache-airflow-providers-common-messaging>=2.0.0", # Set from MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py "apache-airflow-providers-common-sql>=1.18.0", @@ -1148,6 +1152,8 @@ mypy_path = [ "$MYPY_CONFIG_FILE_DIR/providers/common/ai/tests", "$MYPY_CONFIG_FILE_DIR/providers/common/compat/src", "$MYPY_CONFIG_FILE_DIR/providers/common/compat/tests", + "$MYPY_CONFIG_FILE_DIR/providers/common/dataquality/src", + "$MYPY_CONFIG_FILE_DIR/providers/common/dataquality/tests", "$MYPY_CONFIG_FILE_DIR/providers/common/io/src", "$MYPY_CONFIG_FILE_DIR/providers/common/io/tests", "$MYPY_CONFIG_FILE_DIR/providers/common/messaging/src", @@ -1460,6 +1466,7 @@ apache-airflow-providers-cncf-kubernetes = false apache-airflow-providers-cohere = false apache-airflow-providers-common-ai = false apache-airflow-providers-common-compat = false +apache-airflow-providers-common-dataquality = false apache-airflow-providers-common-io = false apache-airflow-providers-common-messaging = false apache-airflow-providers-common-sql = false @@ -1605,6 +1612,7 @@ apache-airflow-providers-cncf-kubernetes = false apache-airflow-providers-cohere = false apache-airflow-providers-common-ai = false apache-airflow-providers-common-compat = false +apache-airflow-providers-common-dataquality = false apache-airflow-providers-common-io = false apache-airflow-providers-common-messaging = false apache-airflow-providers-common-sql = false @@ -1765,6 +1773,7 @@ apache-airflow-providers-cncf-kubernetes = { workspace = true } apache-airflow-providers-cohere = { workspace = true } apache-airflow-providers-common-ai = { workspace = true } apache-airflow-providers-common-compat = { workspace = true } +apache-airflow-providers-common-dataquality = { workspace = true } apache-airflow-providers-common-io = { workspace = true } apache-airflow-providers-common-messaging = { workspace = true } apache-airflow-providers-common-sql = { workspace = true } @@ -1905,6 +1914,7 @@ members = [ "providers/cohere", "providers/common/ai", "providers/common/compat", + "providers/common/dataquality", "providers/common/io", "providers/common/messaging", "providers/common/sql", diff --git a/scripts/ci/docker-compose/remove-sources.yml b/scripts/ci/docker-compose/remove-sources.yml index ed2e4b1bed789..edb644a179a19 100644 --- a/scripts/ci/docker-compose/remove-sources.yml +++ b/scripts/ci/docker-compose/remove-sources.yml @@ -58,6 +58,7 @@ services: - ../../../empty:/opt/airflow/providers/cohere/src - ../../../empty:/opt/airflow/providers/common/ai/src - ../../../empty:/opt/airflow/providers/common/compat/src + - ../../../empty:/opt/airflow/providers/common/dataquality/src - ../../../empty:/opt/airflow/providers/common/io/src - ../../../empty:/opt/airflow/providers/common/messaging/src - ../../../empty:/opt/airflow/providers/common/sql/src diff --git a/scripts/ci/docker-compose/tests-sources.yml b/scripts/ci/docker-compose/tests-sources.yml index 1f6e47ba961c2..eb700548346da 100644 --- a/scripts/ci/docker-compose/tests-sources.yml +++ b/scripts/ci/docker-compose/tests-sources.yml @@ -71,6 +71,7 @@ services: - ../../../providers/cohere/tests:/opt/airflow/providers/cohere/tests - ../../../providers/common/ai/tests:/opt/airflow/providers/common/ai/tests - ../../../providers/common/compat/tests:/opt/airflow/providers/common/compat/tests + - ../../../providers/common/dataquality/tests:/opt/airflow/providers/common/dataquality/tests - ../../../providers/common/io/tests:/opt/airflow/providers/common/io/tests - ../../../providers/common/messaging/tests:/opt/airflow/providers/common/messaging/tests - ../../../providers/common/sql/tests:/opt/airflow/providers/common/sql/tests diff --git a/uv.lock b/uv.lock index 2973c8da2681a..aeb4d3f9de6eb 100644 --- a/uv.lock +++ b/uv.lock @@ -52,6 +52,7 @@ apache-airflow-providers-telegram = false apache-airflow-providers-celery = false apache-airflow-providers-docker = false apache-airflow-providers-sendgrid = false +apache-airflow-providers-common-dataquality = false apache-airflow-providers-common-ai = false apache-airflow = false apache-airflow-shared-observability = false @@ -208,6 +209,7 @@ members = [ "apache-airflow-providers-common-sql", "apache-airflow-providers-databricks", "apache-airflow-providers-datadog", + "apache-airflow-providers-common-dataquality", "apache-airflow-providers-dbt-cloud", "apache-airflow-providers-dingding", "apache-airflow-providers-discord", @@ -1037,6 +1039,7 @@ all = [ { name = "apache-airflow-providers-common-sql", extra = ["pandas", "polars"] }, { name = "apache-airflow-providers-databricks" }, { name = "apache-airflow-providers-datadog" }, + { name = "apache-airflow-providers-common-dataquality" }, { name = "apache-airflow-providers-dbt-cloud" }, { name = "apache-airflow-providers-dingding" }, { name = "apache-airflow-providers-discord" }, @@ -1233,6 +1236,9 @@ databricks = [ datadog = [ { name = "apache-airflow-providers-datadog" }, ] +common-dataquality = [ + { name = "apache-airflow-providers-common-dataquality" }, +] dbt-cloud = [ { name = "apache-airflow-providers-dbt-cloud" }, ] @@ -1642,6 +1648,8 @@ requires-dist = [ { name = "apache-airflow-providers-databricks", marker = "extra == 'databricks'", editable = "providers/databricks" }, { name = "apache-airflow-providers-datadog", marker = "extra == 'all'", editable = "providers/datadog" }, { name = "apache-airflow-providers-datadog", marker = "extra == 'datadog'", editable = "providers/datadog" }, + { name = "apache-airflow-providers-common-dataquality", marker = "extra == 'all'", editable = "providers/common/dataquality" }, + { name = "apache-airflow-providers-common-dataquality", marker = "extra == 'common-dataquality'", editable = "providers/common/dataquality" }, { name = "apache-airflow-providers-dbt-cloud", marker = "extra == 'all'", editable = "providers/dbt/cloud" }, { name = "apache-airflow-providers-dbt-cloud", marker = "extra == 'dbt-cloud'", editable = "providers/dbt/cloud" }, { name = "apache-airflow-providers-dingding", marker = "extra == 'all'", editable = "providers/dingding" }, @@ -1788,7 +1796,7 @@ requires-dist = [ { name = "sentry-sdk", marker = "extra == 'sentry'", specifier = ">=2.30.0" }, { name = "uv", marker = "extra == 'uv'", specifier = ">=0.11.26" }, ] -provides-extras = ["all-core", "async", "graphviz", "gunicorn", "kerberos", "memray", "otel", "statsd", "all-task-sdk", "airbyte", "akeyless", "alibaba", "amazon", "anthropic", "apache-cassandra", "apache-drill", "apache-druid", "apache-flink", "apache-hdfs", "apache-hive", "apache-iceberg", "apache-impala", "apache-kafka", "apache-kylin", "apache-livy", "apache-pig", "apache-pinot", "apache-spark", "apache-tinkerpop", "apprise", "arangodb", "asana", "atlassian-jira", "celery", "clickhousedb", "cloudant", "cncf-kubernetes", "cohere", "common-ai", "common-compat", "common-io", "common-messaging", "common-sql", "databricks", "datadog", "dbt-cloud", "dingding", "discord", "docker", "edge3", "elasticsearch", "exasol", "fab", "facebook", "ftp", "git", "github", "google", "grpc", "hashicorp", "http", "ibm-mq", "imap", "influxdb", "informatica", "jdbc", "jenkins", "keycloak", "microsoft-azure", "microsoft-mssql", "microsoft-psrp", "microsoft-winrm", "mongo", "mysql", "neo4j", "odbc", "openai", "openfaas", "openlineage", "opensearch", "opsgenie", "oracle", "pagerduty", "papermill", "pgvector", "pinecone", "postgres", "presto", "qdrant", "redis", "salesforce", "samba", "segment", "sendgrid", "sftp", "singularity", "slack", "smtp", "snowflake", "sqlite", "ssh", "standard", "tableau", "telegram", "teradata", "trino", "vertica", "vespa", "weaviate", "yandex", "ydb", "zendesk", "all", "aiobotocore", "apache-atlas", "apache-webhdfs", "amazon-aws-auth", "cloudpickle", "github-enterprise", "google-auth", "ldap", "pandas", "polars", "rabbitmq", "sentry", "s3fs", "uv"] +provides-extras = ["all-core", "async", "graphviz", "gunicorn", "kerberos", "memray", "otel", "statsd", "all-task-sdk", "airbyte", "akeyless", "alibaba", "amazon", "anthropic", "apache-cassandra", "apache-drill", "apache-druid", "apache-flink", "apache-hdfs", "apache-hive", "apache-iceberg", "apache-impala", "apache-kafka", "apache-kylin", "apache-livy", "apache-pig", "apache-pinot", "apache-spark", "apache-tinkerpop", "apprise", "arangodb", "asana", "atlassian-jira", "celery", "clickhousedb", "cloudant", "cncf-kubernetes", "cohere", "common-ai", "common-compat", "common-io", "common-messaging", "common-sql", "databricks", "datadog", "common-dataquality", "dbt-cloud", "dingding", "discord", "docker", "edge3", "elasticsearch", "exasol", "fab", "facebook", "ftp", "git", "github", "google", "grpc", "hashicorp", "http", "ibm-mq", "imap", "influxdb", "informatica", "jdbc", "jenkins", "keycloak", "microsoft-azure", "microsoft-mssql", "microsoft-psrp", "microsoft-winrm", "mongo", "mysql", "neo4j", "odbc", "openai", "openfaas", "openlineage", "opensearch", "opsgenie", "oracle", "pagerduty", "papermill", "pgvector", "pinecone", "postgres", "presto", "qdrant", "redis", "salesforce", "samba", "segment", "sendgrid", "sftp", "singularity", "slack", "smtp", "snowflake", "sqlite", "ssh", "standard", "tableau", "telegram", "teradata", "trino", "vertica", "vespa", "weaviate", "yandex", "ydb", "zendesk", "all", "aiobotocore", "apache-atlas", "apache-webhdfs", "amazon-aws-auth", "cloudpickle", "github-enterprise", "google-auth", "ldap", "pandas", "polars", "rabbitmq", "sentry", "s3fs", "uv"] [package.metadata.requires-dev] ci-image = [ @@ -4899,6 +4907,37 @@ dev = [ ] docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] +[[package]] +name = "apache-airflow-providers-common-dataquality" +version = "0.1.0" +source = { editable = "providers/common/dataquality" } +dependencies = [ + { name = "apache-airflow" }, +] + +[package.dev-dependencies] +dev = [ + { name = "apache-airflow" }, + { name = "apache-airflow-devel-common" }, + { name = "apache-airflow-task-sdk" }, +] +docs = [ + { name = "apache-airflow-devel-common", extra = ["docs"] }, +] + +[package.metadata] +requires-dist = [ + { name = "apache-airflow", editable = "." }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "apache-airflow", editable = "." }, + { name = "apache-airflow-devel-common", editable = "devel-common" }, + { name = "apache-airflow-task-sdk", editable = "task-sdk" }, +] +docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] + [[package]] name = "apache-airflow-providers-dbt-cloud" version = "4.9.2"