"""Test suite for Airflow tests"""

test_loader = unittest.TestLoader()

test_suite = test_loader.discover(os.path.join(my_dir, 'tests'), pattern='test_*.py')

"""

Command to tidy up the project root.

Registered as cmdclass in setup() so it can be called with ``python setup.py extra_clean``.

"""

description = "Tidy up the project root"

user_options: List[str] = []

def initialize_options(self) -> None:

"""Set default values for options."""

def finalize_options(self) -> None:

"""Set final values for options."""

@staticmethod

def rm_all_files(files: List[str]) -> None:

"""Remove all files from the list"""

for file in files:

try:

os.remove(file)

except Exception as e:

logger.warning("Error when removing %s: %s", file, e)

def run(self) -> None:

"""Remove temporary files and directories."""

os.chdir(my_dir)

self.rm_all_files(glob.glob('./build/*'))

self.rm_all_files(glob.glob('./**/__pycache__/*', recursive=True))

self.rm_all_files(glob.glob('./**/*.pyc', recursive=True))

self.rm_all_files(glob.glob('./dist/*'))

self.rm_all_files(glob.glob('./*.egg-info'))

self.rm_all_files(glob.glob('./docker-context-files/*.whl'))

"""

Compile and build the frontend assets using yarn and webpack.

Registered as cmdclass in setup() so it can be called with ``python setup.py compile_assets``.

"""

description = "Compile and build the frontend assets"

user_options: List[str] = []

def initialize_options(self) -> None:

"""Set default values for options."""

def finalize_options(self) -> None:

"""Set final values for options."""

def run(self) -> None:

"""Run a command to compile and build assets."""

"""

List all available extras

Registered as cmdclass in setup() so it can be called with ``python setup.py list_extras``.

"""

description = "List available extras"

user_options: List[str] = []

def initialize_options(self) -> None:

"""Set default values for options."""

def finalize_options(self) -> None:

"""Set final values for options."""

def run(self) -> None:

"""List extras."""

"""

Return a version to identify the state of the underlying git repo. The version will

indicate whether the head of the current git-backed working directory is tied to a

release tag or not : it will indicate the former with a 'release:{version}' prefix

and the latter with a '.dev0' suffix. Following the prefix will be a sha of the current

branch head. Finally, a "dirty" suffix is appended to indicate that uncommitted

changes are present.

:param str version_: Semver version

:return: Found Airflow version in Git repo

:rtype: str

"""

try:

import git

try:

repo = git.Repo(os.path.join(*[my_dir, '.git']))

except git.NoSuchPathError:

logger.warning('.git directory not found: Cannot compute the git version')

return ''

except git.InvalidGitRepositoryError:

logger.warning('Invalid .git directory not found: Cannot compute the git version')

return ''

except ImportError:

logger.warning('gitpython not found: Cannot compute the git version.')

return ''

if repo:

sha = repo.head.commit.hexsha

if repo.is_dirty():

return f'.dev0+{sha}.dirty'

# commit is clean

return f'.release:{version_}+{sha}'

"""

Write the Semver version + git hash to file, e.g. ".dev0+2f635dc265e78db6708f59f68e8009abb92c1e65".

:param str filename: Destination file to write

"""

text = f"{git_version(version)}"

with open(filename, 'w') as file:

'boto3>=1.15.0,<1.19.0',

'watchtower~=2.0.1',

'jsonpath_ng>=1.5.3',

'redshift_connector~=2.0.888',

'sqlalchemy_redshift~=0.8.6',

pandas_requirement,

'eventlet>= 0.9.7',

'gevent>=0.13',

'greenlet>=0.4.9',

'azure-batch>=8.0.0',

'azure-cosmos>=4.0.0,<5',

'azure-datalake-store>=0.0.45',

'azure-identity>=1.3.1',

'azure-keyvault>=4.1.0',

'azure-kusto-data>=0.0.43,<0.1',

'azure-mgmt-containerinstance>=1.5.0,<2.0',

'azure-mgmt-datafactory>=1.0.0,<2.0',

'azure-mgmt-datalake-store>=0.5.0',

'azure-mgmt-resource>=2.2.0',

# limited due to https://github.com/Azure/azure-sdk-for-python/pull/18801 implementation released in 12.9

'azure-storage-blob>=12.7.0,<12.9.0',

'azure-storage-common>=2.1.0',

'azure-storage-file>=2.1.0',

'cloudpickle>=1.4.1, <1.5.0',

'dask<2021.3.1;python_version<"3.7"', # dask stopped supporting python 3.6 in 2021.3.1 version

'dask>=2.9.0, <2021.6.1;python_version>="3.7"', # dask 2021.6.1 does not work with `distributed`

'distributed>=2.11.1, <2.20',

'click>=7.1,<9',

# Sphinx is limited to < 3.5.0 because of https://github.com/sphinx-doc/sphinx/issues/8880

'sphinx>=2.1.2, <3.5.0',

'sphinx-airflow-theme',

'sphinx-argparse>=0.1.13',

'sphinx-autoapi==1.0.0',

'sphinx-copybutton',

'sphinx-jinja~=1.1',

'sphinx-rtd-theme>=0.1.6',

'sphinxcontrib-httpdomain>=1.7.0',

'sphinxcontrib-redoc>=1.6.0',

'sphinxcontrib-spelling==7.2.1',

'elasticsearch>7',

'elasticsearch-dbapi',

'elasticsearch-dsl>=5.0.0',

'PyOpenSSL',

# The Google Ads 14.0.1 breaks PIP and eager upgrade as it requires

# google-api-core>=2.0.0 which cannot be used yet (see below comment)

# and https://github.com/apache/airflow/issues/18705#issuecomment-933746150

'google-ads>=12.0.0,<14.0.1',

# Maintainers, please do not require google-api-core>=2.x.x

# Until this issue is closed

# https://github.com/googleapis/google-cloud-python/issues/10566

'google-api-core>=1.25.1,<3.0.0',

'google-api-python-client>=1.6.0,<2.0.0',

# Maintainers, please do not require google-auth>=2.x.x

# Until this issue is closed

# https://github.com/googleapis/google-cloud-python/issues/10566

'google-auth>=1.0.0,<3.0.0',

'google-auth-httplib2>=0.0.1',

'google-cloud-automl>=2.1.0,<3.0.0',

'google-cloud-bigquery-datatransfer>=3.0.0,<4.0.0',

'google-cloud-bigtable>=1.0.0,<2.0.0',

'google-cloud-build>=3.0.0,<4.0.0',

'google-cloud-container>=0.1.1,<2.0.0',

'google-cloud-datacatalog>=3.0.0,<4.0.0',

'google-cloud-dataproc>=3.1.0,<4.0.0',

'google-cloud-dataproc-metastore>=1.2.0,<2.0.0',

'google-cloud-dlp>=0.11.0,<2.0.0',

'google-cloud-kms>=2.0.0,<3.0.0',

'google-cloud-language>=1.1.1,<2.0.0',

'google-cloud-logging>=2.1.1,<3.0.0',

# 1.1.0 removed field_mask and broke import for released providers

# We can remove the <1.1.0 limitation after we release new Google Provider

'google-cloud-memcache>=0.2.0,<1.1.0',

'google-cloud-monitoring>=2.0.0,<3.0.0',

'google-cloud-os-login>=2.0.0,<3.0.0',

'google-cloud-pubsub>=2.0.0,<3.0.0',

'google-cloud-redis>=2.0.0,<3.0.0',

'google-cloud-secret-manager>=0.2.0,<2.0.0',

'google-cloud-spanner>=1.10.0,<2.0.0',

'google-cloud-speech>=0.36.3,<2.0.0',

'google-cloud-storage>=1.30,<2.0.0',

'google-cloud-tasks>=2.0.0,<3.0.0',

'google-cloud-texttospeech>=0.4.0,<2.0.0',

'google-cloud-translate>=1.5.0,<2.0.0',

'google-cloud-videointelligence>=1.7.0,<2.0.0',

'google-cloud-vision>=0.35.2,<2.0.0',

'google-cloud-workflows>=0.1.0,<2.0.0',

'grpcio-gcp>=0.2.2',

'httpx',

'json-merge-patch~=0.2',

# pandas-gbq 0.15.0 release broke google provider's bigquery import

# _check_google_client_version (airflow/providers/google/cloud/hooks/bigquery.py:49)

'pandas-gbq<0.15.0',

pandas_requirement,

'google-auth>=1.0.0, <3.0.0',

'google-auth-httplib2>=0.0.1',

'grpcio>=1.15.0',

'hmsclient>=0.1.0',

'pyhive[hive]>=0.6.0;python_version<"3.9"',

'thrift>=0.9.2',

pandas_requirement,

# The 2.26.0 release of requests got rid of the chardet LGPL mandatory dependency, allowing us to

# release it as a requirement for airflow

'requests>=2.26.0',

'pykerberos>=1.1.13',

'requests_kerberos>=0.10.0',

'thrift_sasl>=0.2.0',

# pinotdb v0.1.1 may still work with older versions of Apache Pinot, but we've confirmed that it

# causes a problem with newer versions.

'pinotdb>0.1.2,<1.0.0',

'paramiko>=2.6.0',

'pysftp>=0.2.9',

'sshtunnel>=0.3.2,<0.5',

'mypy==0.910',

'types-croniter',

'types-docutils',

'types-freezegun',

'types-paramiko',

'types-protobuf',

'types-python-dateutil',

'types-python-slugify',

'types-pytz',

'types-redis',

'types-requests',

'types-setuptools',

'types-termcolor',

'types-tabulate',

'types-Markdown',

'types-PyMySQL',

'types-PyYAML',

'aws_xray_sdk',

'beautifulsoup4~=4.7.1',

'black',

'blinker',

'bowler',

'click>=7.1,<9',

'coverage',

'filelock',

'flake8>=3.6.0',

'flake8-colors',

'flaky',

'freezegun',

'github3.py',

'gitpython',

'ipdb',

'jira',

'jsondiff',

'mongomock',

'moto~=2.2, >=2.2.12',

'parameterized',

'paramiko',

'pipdeptree',

'pre-commit',

'pypsrp',

'pygithub',

'pysftp',

'pytest~=6.0',

'pytest-asyncio',

'pytest-cov',

'pytest-instafail',

'pytest-rerunfailures~=9.1',

'pytest-timeouts',

'pytest-xdist',

'python-jose',

'pywinrm',

'qds-sdk>=1.9.6',

'pytest-httpx',

'requests_mock',

'semver',

'wheel',

'yamllint',

'airbyte': http_provider,

'alibaba': alibaba,

'amazon': amazon,

'apache.beam': apache_beam,

'apache.cassandra': cassandra,

'apache.drill': drill,

'apache.druid': druid,

'apache.hdfs': hdfs,

'apache.hive': hive,

'apache.kylin': kylin,

'apache.livy': http_provider,

'apache.pig': [],

'apache.pinot': pinot,

'apache.spark': spark,

'apache.sqoop': [],

'asana': asana,

'celery': celery,

'cloudant': cloudant,

'cncf.kubernetes': kubernetes,

'databricks': databricks,

'datadog': datadog,

'dingding': [],

'discord': [],

'docker': docker,

'elasticsearch': elasticsearch,

'exasol': exasol,

'facebook': facebook,

'ftp': [],

'google': google,

'grpc': grpc,

'hashicorp': hashicorp,

'http': http,

'imap': [],

'influxdb': influxdb,

'jdbc': jdbc,

'jenkins': jenkins,

'jira': jira,

'microsoft.azure': azure,

'microsoft.mssql': mssql,

'microsoft.psrp': psrp,

'microsoft.winrm': winrm,

'mongo': mongo,

'mysql': mysql,

'neo4j': neo4j,

'odbc': odbc,

'openfaas': [],

'opsgenie': http_provider,

'oracle': oracle,

'pagerduty': pagerduty,

'papermill': papermill,

'plexus': plexus,

'postgres': postgres,

'presto': presto,

'qubole': qubole,

'redis': redis,

'salesforce': salesforce,

'samba': samba,

'segment': segment,

'sendgrid': sendgrid,

'sftp': ssh,

'singularity': singularity,

'slack': slack,

'snowflake': snowflake,

'sqlite': [],

'ssh': ssh,

'tableau': tableau,

'telegram': telegram,

'trino': trino,

'vertica': vertica,

'yandex': yandex,

'zendesk': zendesk,

'async': async_packages,

'celery': celery, # also has provider, but it extends the core with the Celery executor

'cgroups': cgroups,

'cncf.kubernetes': kubernetes, # also has provider, but it extends the core with the KubernetesExecutor

'dask': dask,

'deprecated_api': deprecated_api,

'github_enterprise': flask_appbuilder_authlib,

'google_auth': flask_appbuilder_authlib,

'kerberos': kerberos,

'ldap': ldap,

'leveldb': leveldb,

'pandas': pandas,

'password': password,

'rabbitmq': rabbitmq,

'sentry': sentry,

'statsd': statsd,

'virtualenv': virtualenv,

"""

Adds extras for all providers.

By default all providers have the same extra name as provider id, for example

'apache.hive' extra has 'apache.hive' provider requirement.

"""

for provider_name, provider_requirement in PROVIDERS_REQUIREMENTS.items():

'atlas': 'apache.atlas',

'aws': 'amazon',

'azure': 'microsoft.azure',

'cassandra': 'apache.cassandra',

'crypto': '', # All crypto requirements are installation requirements of core Airflow

'druid': 'apache.druid',

'gcp': 'google',

'gcp_api': 'google',

'hdfs': 'apache.hdfs',

'hive': 'apache.hive',

'kubernetes': 'cncf.kubernetes',

'mssql': 'microsoft.mssql',

'pinot': 'apache.pinot',

'qds': 'qubole',

's3': 'amazon',

'spark': 'apache.spark',

'webhdfs': 'apache.webhdfs',

'winrm': 'microsoft.winrm',

"""

Add extras for all deprecated aliases. Requirements for those deprecated aliases are the same

as the extras they are replaced with.

The requirements are not copies - those are the same lists as for the new extras. This is intended.

Thanks to that if the original extras are later extended with providers, aliases are extended as well.

"""

for alias, extra in EXTRAS_DEPRECATED_ALIASES.items():

requirements = EXTRAS_REQUIREMENTS.get(extra) if extra != '' else []

if requirements is None:

raise Exception(f"The extra {extra} is missing for deprecated alias {alias}")

"""

For deprecated aliases that are providers, we will swap the providers requirements to instead

be the provider itself.

e.g. {"kubernetes": ["kubernetes>=3.0.0, <12.0.0", ...]} becomes

{"kubernetes": ["apache-airflow-provider-cncf-kubernetes"]}

"""

for alias, provider in EXTRAS_DEPRECATED_ALIASES.items():

if alias in EXTRAS_DEPRECATED_ALIASES_NOT_PROVIDERS:

continue

'apache.cassandra',

'apache.drill',

'apache.druid',

'apache.hdfs',

'apache.hive',

'apache.pinot',

'cloudant',

'exasol',

'influxdb',

'microsoft.mssql',

'mongo',

'mysql',

'neo4j',

'postgres',

'presto',

'trino',

'vertica',

"""

Checks if package should be excluded.

:param package: package name (beginning of it)

:param exclusion_list: list of excluded packages

:return: true if package should be excluded

"""

package

for package in devel_all

if not is_package_excluded(package=package, exclusion_list=PACKAGES_EXCLUDED_FOR_ALL)

"""

For Python 3.6+ the dictionary order remains when keys() are retrieved.

Sort both: extras and list of dependencies to make it easier to analyse problems

external packages will be first, then if providers are added they are added at the end of the lists.

"""

sorted_requirements = dict(sorted(EXTRAS_REQUIREMENTS.items()))

for extra_list in sorted_requirements.values():

extra_list.sort()

'ftp',

'http',

'imap',

'sqlite',

"""

Builds the name of provider package out of the package id provided/

:param package_id: id of the package (like amazon or microsoft.azure)

:return: full name of package in PyPI

"""

package_suffix = package_id.replace(".", "-")

"""

Returns packages excluded for the current python version.

Currently the only excluded provider is apache hive for Python 3.9.

Until https://github.com/dropbox/PyHive/issues/380 is fixed.

"""

"""Returns all provider packages configured in setup.py"""

excluded_providers = get_excluded_providers()

return " ".join(

get_provider_package_from_package_id(package)

for package in PROVIDERS_REQUIREMENTS

if package not in excluded_providers

"""The setuptools.Distribution subclass with Airflow specific behaviour"""

def __init__(self, attrs=None):

super().__init__(attrs)

self.install_requires = None

def parse_config_files(self, *args, **kwargs) -> None:

"""

Ensure that when we have been asked to install providers from sources

that we don't *also* try to install those providers from PyPI.

Also we should make sure that in this case we copy provider.yaml files so that

Providers manager can find package information.

"""

super().parse_config_files(*args, **kwargs)

if os.getenv(INSTALL_PROVIDERS_FROM_SOURCES) == 'true':

self.install_requires = [

req for req in self.install_requires if not req.startswith('apache-airflow-providers-')

]

provider_yaml_files = glob.glob("airflow/providers/**/provider.yaml", recursive=True)

for provider_yaml_file in provider_yaml_files:

provider_relative_path = relpath(provider_yaml_file, os.path.join(my_dir, "airflow"))

self.package_data['airflow'].append(provider_relative_path)

else:

self.install_requires.extend(

[get_provider_package_from_package_id(package_id) for package_id in PREINSTALLED_PROVIDERS]

"""

Replaces extra requirement with provider package. The intention here is that when

the provider is added as dependency of extra, there is no need to add the dependencies

separately. This is not needed and even harmful, because in case of future versions of

the provider, the requirements might change, so hard-coding requirements from the version

that was available at the release time might cause dependency conflicts in the future.

Say for example that you have salesforce provider with those deps:

{ 'salesforce': ['simple-salesforce>=1.0.0', 'tableauserverclient'] }

Initially ['salesforce'] extra has those requirements and it works like that when you install

it when INSTALL_PROVIDERS_FROM_SOURCES is set to `true` (during the development). However, when

the production installation is used, The dependencies are changed:

{ 'salesforce': ['apache-airflow-providers-salesforce'] }

And then, 'apache-airflow-providers-salesforce' package has those 'install_requires' dependencies:

['simple-salesforce>=1.0.0', 'tableauserverclient']

So transitively 'salesforce' extra has all the requirements it needs and in case the provider

changes it's dependencies, they will transitively change as well.

In the constraint mechanism we save both - provider versions and it's dependencies

version, which means that installation using constraints is repeatable.

:param extra: Name of the extra to add providers to

:param providers: list of provider ids

"""

EXTRAS_REQUIREMENTS[extra] = [

get_provider_package_from_package_id(package_name) for package_name in providers

"""

Adds provider packages as requirements to extra. This is used to add provider packages as requirements

to the "bulk" kind of extras. Those bulk extras do not have the detailed 'extra' requirements as

initial values, so instead of replacing them (see previous function) we can extend them.

:param extra: Name of the extra to add providers to

:param providers: list of provider ids

"""

EXTRAS_REQUIREMENTS[extra].extend(

[get_provider_package_from_package_id(package_name) for package_name in providers]

"""

In case of regular installation (providers installed from packages), we should add extra dependencies to

Airflow - to get the providers automatically installed when those extras are installed.

For providers installed from sources we skip that step. That helps to test and install airflow with