From c012e4b33ed43f29b9e068c2201e7ff32adde4f8 Mon Sep 17 00:00:00 2001 From: gpupo Date: Tue, 1 Apr 2025 10:49:59 -0300 Subject: [PATCH 01/13] mode first dag to subdir --- dag/{ => simple_dag}/BUILD.bazel | 0 dag/{ => simple_dag}/__init__.py | 0 dag/{ => simple_dag}/local_dev.py | 0 dag/{ => simple_dag}/minimal_dag.py | 0 dag/{ => simple_dag}/test_dag.py | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename dag/{ => simple_dag}/BUILD.bazel (100%) rename dag/{ => simple_dag}/__init__.py (100%) rename dag/{ => simple_dag}/local_dev.py (100%) rename dag/{ => simple_dag}/minimal_dag.py (100%) rename dag/{ => simple_dag}/test_dag.py (100%) diff --git a/dag/BUILD.bazel b/dag/simple_dag/BUILD.bazel similarity index 100% rename from dag/BUILD.bazel rename to dag/simple_dag/BUILD.bazel diff --git a/dag/__init__.py b/dag/simple_dag/__init__.py similarity index 100% rename from dag/__init__.py rename to dag/simple_dag/__init__.py diff --git a/dag/local_dev.py b/dag/simple_dag/local_dev.py similarity index 100% rename from dag/local_dev.py rename to dag/simple_dag/local_dev.py diff --git a/dag/minimal_dag.py b/dag/simple_dag/minimal_dag.py similarity index 100% rename from dag/minimal_dag.py rename to dag/simple_dag/minimal_dag.py diff --git a/dag/test_dag.py b/dag/simple_dag/test_dag.py similarity index 100% rename from dag/test_dag.py rename to dag/simple_dag/test_dag.py From 494a0c8899ce39cb6dbb82db81726f7821d8d9e5 Mon Sep 17 00:00:00 2001 From: gpupo Date: Tue, 1 Apr 2025 10:50:10 -0300 Subject: [PATCH 02/13] dag root build file --- dag/BUILD.bazel | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 dag/BUILD.bazel diff --git a/dag/BUILD.bazel b/dag/BUILD.bazel new file mode 100644 index 0000000..e69de29 From 856abf5d0c82d5bb555a2520743542f898e78aa2 Mon Sep 17 00:00:00 2001 From: gpupo Date: Tue, 1 Apr 2025 10:51:40 -0300 Subject: [PATCH 03/13] doc with subdir --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index dccc5f3..f279ef6 100644 --- a/README.md +++ b/README.md @@ -50,11 +50,11 @@ bazel test //... Rodando os testes unitarios - bazel run //dag:dag_test + bazel run //dag/simple_dag:dag_test Rodando a adaptacao para ambiente local - bazel run //dag:dag_dev + bazel run //dag/simple_dag:dag_dev TODO: Adicionar DockerOperator From 86d65420da4d9193b8281bbc1c6ca85d75798d2b Mon Sep 17 00:00:00 2001 From: gpupo Date: Tue, 1 Apr 2025 17:21:11 -0300 Subject: [PATCH 04/13] add https://github.com/bazel-contrib/rules_oci/ --- MODULE.bazel | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/MODULE.bazel b/MODULE.bazel index 523e018..5c5b268 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -6,9 +6,12 @@ module( bazel_dep(name = "rules_python", version = "1.3.0") -# SEE: https://github.com/theoremlp/rules_uv/ +# INFO: https://github.com/theoremlp/rules_uv/ bazel_dep(name = "rules_uv", version = "0.63.0") +# INFO: https://github.com/bazel-contrib/rules_oci/ +bazel_dep(name = "rules_oci", version = "2.2.5") + python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain( python_version = "3.12", From 67091ab6a7048cdea9146ad304925bc1d90b7787 Mon Sep 17 00:00:00 2001 From: gpupo Date: Tue, 1 Apr 2025 17:27:00 -0300 Subject: [PATCH 05/13] module doc --- MODULE.bazel | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/MODULE.bazel b/MODULE.bazel index 5c5b268..6dc71c2 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -1,6 +1,6 @@ module( name = "bazel-simple-python", - version = "0.2", + version = "0.3", compatibility_level = 1, ) @@ -12,6 +12,7 @@ bazel_dep(name = "rules_uv", version = "0.63.0") # INFO: https://github.com/bazel-contrib/rules_oci/ bazel_dep(name = "rules_oci", version = "2.2.5") +# INFO: Python config python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain( python_version = "3.12", @@ -39,3 +40,14 @@ pip.parse( requirements_lock = "//:requirements.txt", ) use_repo(pip, "pypi") + +# INFO: Docker Images +oci = use_extension("@rules_oci//oci:extensions.bzl", "oci") +oci.pull( + name = "python_3_12_9_image", + #digest = "sha256:...", # INFO: https://hub.docker.com/_/python + image = "index.docker.io/library/python", + platforms = ["linux/amd64"], + tag = "3.12.9-slim", +) +use_repo(oci, "python_3_12_9_image") From e28811242ce1a902b11a0a834191f072d84b5941 Mon Sep 17 00:00:00 2001 From: gpupo Date: Tue, 1 Apr 2025 17:27:45 -0300 Subject: [PATCH 06/13] New Dag for container --- dag/docker-operator-dag/Dockerfile | 9 +++++++++ dag/docker-operator-dag/__init__.py | 0 2 files changed, 9 insertions(+) create mode 100644 dag/docker-operator-dag/Dockerfile create mode 100644 dag/docker-operator-dag/__init__.py diff --git a/dag/docker-operator-dag/Dockerfile b/dag/docker-operator-dag/Dockerfile new file mode 100644 index 0000000..59fb396 --- /dev/null +++ b/dag/docker-operator-dag/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.12.9-slim + +WORKDIR /app +COPY task.py /app/ + +# Instala dependências (se necessário) +RUN pip install --no-cache-dir pandas numpy # Exemplo + +CMD ["python", "task.py"] diff --git a/dag/docker-operator-dag/__init__.py b/dag/docker-operator-dag/__init__.py new file mode 100644 index 0000000..e69de29 From 15577f719b13877f1dfdcef714dfd3e4120931b5 Mon Sep 17 00:00:00 2001 From: gpupo Date: Tue, 1 Apr 2025 20:59:26 -0300 Subject: [PATCH 07/13] add module dep rules_pkg --- MODULE.bazel | 14 +- MODULE.bazel.lock | 610 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 617 insertions(+), 7 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 6dc71c2..8ee0da5 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -12,6 +12,9 @@ bazel_dep(name = "rules_uv", version = "0.63.0") # INFO: https://github.com/bazel-contrib/rules_oci/ bazel_dep(name = "rules_oci", version = "2.2.5") +# INFO:https://github.com/bazelbuild/rules_pkg/ +bazel_dep(name = "rules_pkg", version = "1.1.0") + # INFO: Python config python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain( @@ -43,11 +46,12 @@ use_repo(pip, "pypi") # INFO: Docker Images oci = use_extension("@rules_oci//oci:extensions.bzl", "oci") + +# INFO: https://hub.docker.com/_/python oci.pull( - name = "python_3_12_9_image", - #digest = "sha256:...", # INFO: https://hub.docker.com/_/python - image = "index.docker.io/library/python", + name = "python3", + digest = "sha256:e52ca5f579cc58fed41efcbb55a0ed5dccf6c7a156cba76acfb4ab42fc19dd00", + image = "python:3.9-slim", platforms = ["linux/amd64"], - tag = "3.12.9-slim", ) -use_repo(oci, "python_3_12_9_image") +use_repo(oci, "python3") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 82fd91f..4014eb1 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -10,7 +10,10 @@ "https://bcr.bazel.build/modules/abseil-cpp/20230802.1/MODULE.bazel": "fa92e2eb41a04df73cdabeec37107316f7e5272650f81d6cc096418fe647b915", "https://bcr.bazel.build/modules/abseil-cpp/20240116.1/MODULE.bazel": "37bcdb4440fbb61df6a1c296ae01b327f19e9bb521f9b8e26ec854b6f97309ed", "https://bcr.bazel.build/modules/abseil-cpp/20240116.1/source.json": "9be551b8d4e3ef76875c0d744b5d6a504a27e3ae67bc6b28f46415fd2d2957da", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "780d1a6522b28f5edb7ea09630748720721dfe27690d65a2d33aa7509de77e07", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.7.2/source.json": "a9ee2898e86eb8106e67b2adcd63de788cd922dd82f90f6775f13b0bf2248d75", "https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd", + "https://bcr.bazel.build/modules/bazel_features/1.10.0/MODULE.bazel": "f75e8807570484a99be90abcd52b5e1f390362c258bcb73106f4544957a48101", "https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8", "https://bcr.bazel.build/modules/bazel_features/1.15.0/MODULE.bazel": "d38ff6e517149dc509406aca0db3ad1efdd890a85e049585b7234d04238e2a4d", "https://bcr.bazel.build/modules/bazel_features/1.17.0/MODULE.bazel": "039de32d21b816b47bd42c778e0454217e9c9caac4a3cf8e15c7231ee3ddee4d", @@ -116,9 +119,12 @@ "https://bcr.bazel.build/modules/rules_license/1.0.0/source.json": "a52c89e54cc311196e478f8382df91c15f7a2bfdf4c6cd0e2675cc2ff0b56efb", "https://bcr.bazel.build/modules/rules_multitool/0.11.0/MODULE.bazel": "8d9dda78d2398e136300d3ef4fbcc89ede7c32c158d8c016fa7d032df41c4aaf", "https://bcr.bazel.build/modules/rules_multitool/0.11.0/source.json": "0b86574a1eaff37c33aafaff095ea16d6ac846beb94ffc74c4fcf626f8f80681", + "https://bcr.bazel.build/modules/rules_oci/2.2.5/MODULE.bazel": "9086fa85e46ca1116317f64e031403ce73eaf214ad9251ee3bdf3199734cf2ac", + "https://bcr.bazel.build/modules/rules_oci/2.2.5/source.json": "4cf66bdafeb0df214d43e0e4b1a442223bca421f62d38650b466cbb001894293", "https://bcr.bazel.build/modules/rules_pkg/0.7.0/MODULE.bazel": "df99f03fc7934a4737122518bb87e667e62d780b610910f0447665a7e2be62dc", "https://bcr.bazel.build/modules/rules_pkg/1.0.1/MODULE.bazel": "5b1df97dbc29623bccdf2b0dcd0f5cb08e2f2c9050aab1092fd39a41e82686ff", - "https://bcr.bazel.build/modules/rules_pkg/1.0.1/source.json": "bd82e5d7b9ce2d31e380dd9f50c111d678c3bdaca190cb76b0e1c71b05e1ba8a", + "https://bcr.bazel.build/modules/rules_pkg/1.1.0/MODULE.bazel": "9db8031e71b6ef32d1846106e10dd0ee2deac042bd9a2de22b4761b0c3036453", + "https://bcr.bazel.build/modules/rules_pkg/1.1.0/source.json": "fef768df13a92ce6067e1cd0cdc47560dace01354f1d921cfb1d632511f7d608", "https://bcr.bazel.build/modules/rules_proto/4.0.0/MODULE.bazel": "a7a7b6ce9bee418c1a760b3d84f83a299ad6952f9903c67f19e4edd964894e06", "https://bcr.bazel.build/modules/rules_proto/5.3.0-21.7/MODULE.bazel": "e8dff86b0971688790ae75528fe1813f71809b5afd57facb44dad9e8eca631b7", "https://bcr.bazel.build/modules/rules_proto/6.0.0-rc1/MODULE.bazel": "1e5b502e2e1a9e825eef74476a5a1ee524a92297085015a052510b09a1a09483", @@ -133,6 +139,7 @@ "https://bcr.bazel.build/modules/rules_python/0.34.0/MODULE.bazel": "1d623d026e075b78c9fde483a889cda7996f5da4f36dffb24c246ab30f06513a", "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c", "https://bcr.bazel.build/modules/rules_python/0.40.0/MODULE.bazel": "9d1a3cd88ed7d8e39583d9ffe56ae8a244f67783ae89b60caafc9f5cf318ada7", + "https://bcr.bazel.build/modules/rules_python/1.0.0/MODULE.bazel": "898a3d999c22caa585eb062b600f88654bf92efb204fa346fb55f6f8edffca43", "https://bcr.bazel.build/modules/rules_python/1.3.0/MODULE.bazel": "8361d57eafb67c09b75bf4bbe6be360e1b8f4f18118ab48037f2bd50aa2ccb13", "https://bcr.bazel.build/modules/rules_python/1.3.0/source.json": "25932f917cd279c7baefa6cb1d3fa8750a7a29de522024449b19af6eab51f4a0", "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", @@ -141,6 +148,7 @@ "https://bcr.bazel.build/modules/rules_uv/0.63.0/source.json": "a35557089c539e62a0da9e4cd1e1af22dbf27d24b9c230ce3567a1f48fc7f4b4", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", + "https://bcr.bazel.build/modules/stardoc/0.5.4/MODULE.bazel": "6569966df04610b8520957cb8e97cf2e9faac2c0309657c537ab51c16c18a2a4", "https://bcr.bazel.build/modules/stardoc/0.5.6/MODULE.bazel": "c43dabc564990eeab55e25ed61c07a1aadafe9ece96a4efabb3f8bf9063b71ef", "https://bcr.bazel.build/modules/stardoc/0.7.0/MODULE.bazel": "05e3d6d30c099b6770e97da986c53bd31844d7f13d41412480ea265ac9e8079c", "https://bcr.bazel.build/modules/stardoc/0.7.1/MODULE.bazel": "3548faea4ee5dda5580f9af150e79d0f6aea934fc60c1cc50f4efdd9420759e7", @@ -155,6 +163,415 @@ }, "selectedYankedVersions": {}, "moduleExtensions": { + "@@aspect_bazel_lib+//lib:extensions.bzl%toolchains": { + "general": { + "bzlTransitiveDigest": "GZqx0/yUW5hj1PczZlZYPEAG7qnnsdcQ+hrQ6CuAK18=", + "usagesDigest": "keQp/SiGDG5Tpkufs1BpUmPED/1BBHtq3tTGeaqk1CY=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "copy_directory_darwin_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_directory_toolchain.bzl%copy_directory_platform_repo", + "attributes": { + "platform": "darwin_amd64" + } + }, + "copy_directory_darwin_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_directory_toolchain.bzl%copy_directory_platform_repo", + "attributes": { + "platform": "darwin_arm64" + } + }, + "copy_directory_freebsd_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_directory_toolchain.bzl%copy_directory_platform_repo", + "attributes": { + "platform": "freebsd_amd64" + } + }, + "copy_directory_linux_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_directory_toolchain.bzl%copy_directory_platform_repo", + "attributes": { + "platform": "linux_amd64" + } + }, + "copy_directory_linux_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_directory_toolchain.bzl%copy_directory_platform_repo", + "attributes": { + "platform": "linux_arm64" + } + }, + "copy_directory_windows_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_directory_toolchain.bzl%copy_directory_platform_repo", + "attributes": { + "platform": "windows_amd64" + } + }, + "copy_directory_toolchains": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_directory_toolchain.bzl%copy_directory_toolchains_repo", + "attributes": { + "user_repository_name": "copy_directory" + } + }, + "copy_to_directory_darwin_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_to_directory_toolchain.bzl%copy_to_directory_platform_repo", + "attributes": { + "platform": "darwin_amd64" + } + }, + "copy_to_directory_darwin_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_to_directory_toolchain.bzl%copy_to_directory_platform_repo", + "attributes": { + "platform": "darwin_arm64" + } + }, + "copy_to_directory_freebsd_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_to_directory_toolchain.bzl%copy_to_directory_platform_repo", + "attributes": { + "platform": "freebsd_amd64" + } + }, + "copy_to_directory_linux_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_to_directory_toolchain.bzl%copy_to_directory_platform_repo", + "attributes": { + "platform": "linux_amd64" + } + }, + "copy_to_directory_linux_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_to_directory_toolchain.bzl%copy_to_directory_platform_repo", + "attributes": { + "platform": "linux_arm64" + } + }, + "copy_to_directory_windows_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_to_directory_toolchain.bzl%copy_to_directory_platform_repo", + "attributes": { + "platform": "windows_amd64" + } + }, + "copy_to_directory_toolchains": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:copy_to_directory_toolchain.bzl%copy_to_directory_toolchains_repo", + "attributes": { + "user_repository_name": "copy_to_directory" + } + }, + "jq_darwin_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:jq_toolchain.bzl%jq_platform_repo", + "attributes": { + "platform": "darwin_amd64", + "version": "1.7" + } + }, + "jq_darwin_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:jq_toolchain.bzl%jq_platform_repo", + "attributes": { + "platform": "darwin_arm64", + "version": "1.7" + } + }, + "jq_linux_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:jq_toolchain.bzl%jq_platform_repo", + "attributes": { + "platform": "linux_amd64", + "version": "1.7" + } + }, + "jq_linux_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:jq_toolchain.bzl%jq_platform_repo", + "attributes": { + "platform": "linux_arm64", + "version": "1.7" + } + }, + "jq_windows_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:jq_toolchain.bzl%jq_platform_repo", + "attributes": { + "platform": "windows_amd64", + "version": "1.7" + } + }, + "jq": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:jq_toolchain.bzl%jq_host_alias_repo", + "attributes": {} + }, + "jq_toolchains": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:jq_toolchain.bzl%jq_toolchains_repo", + "attributes": { + "user_repository_name": "jq" + } + }, + "yq_darwin_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:yq_toolchain.bzl%yq_platform_repo", + "attributes": { + "platform": "darwin_amd64", + "version": "4.25.2" + } + }, + "yq_darwin_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:yq_toolchain.bzl%yq_platform_repo", + "attributes": { + "platform": "darwin_arm64", + "version": "4.25.2" + } + }, + "yq_linux_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:yq_toolchain.bzl%yq_platform_repo", + "attributes": { + "platform": "linux_amd64", + "version": "4.25.2" + } + }, + "yq_linux_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:yq_toolchain.bzl%yq_platform_repo", + "attributes": { + "platform": "linux_arm64", + "version": "4.25.2" + } + }, + "yq_linux_s390x": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:yq_toolchain.bzl%yq_platform_repo", + "attributes": { + "platform": "linux_s390x", + "version": "4.25.2" + } + }, + "yq_linux_ppc64le": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:yq_toolchain.bzl%yq_platform_repo", + "attributes": { + "platform": "linux_ppc64le", + "version": "4.25.2" + } + }, + "yq_windows_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:yq_toolchain.bzl%yq_platform_repo", + "attributes": { + "platform": "windows_amd64", + "version": "4.25.2" + } + }, + "yq": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:yq_toolchain.bzl%yq_host_alias_repo", + "attributes": {} + }, + "yq_toolchains": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:yq_toolchain.bzl%yq_toolchains_repo", + "attributes": { + "user_repository_name": "yq" + } + }, + "coreutils_darwin_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:coreutils_toolchain.bzl%coreutils_platform_repo", + "attributes": { + "platform": "darwin_amd64", + "version": "0.0.23" + } + }, + "coreutils_darwin_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:coreutils_toolchain.bzl%coreutils_platform_repo", + "attributes": { + "platform": "darwin_arm64", + "version": "0.0.23" + } + }, + "coreutils_linux_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:coreutils_toolchain.bzl%coreutils_platform_repo", + "attributes": { + "platform": "linux_amd64", + "version": "0.0.23" + } + }, + "coreutils_linux_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:coreutils_toolchain.bzl%coreutils_platform_repo", + "attributes": { + "platform": "linux_arm64", + "version": "0.0.23" + } + }, + "coreutils_windows_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:coreutils_toolchain.bzl%coreutils_platform_repo", + "attributes": { + "platform": "windows_amd64", + "version": "0.0.23" + } + }, + "coreutils_toolchains": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:coreutils_toolchain.bzl%coreutils_toolchains_repo", + "attributes": { + "user_repository_name": "coreutils" + } + }, + "bsd_tar_darwin_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", + "attributes": { + "platform": "darwin_amd64" + } + }, + "bsd_tar_darwin_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", + "attributes": { + "platform": "darwin_arm64" + } + }, + "bsd_tar_linux_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", + "attributes": { + "platform": "linux_amd64" + } + }, + "bsd_tar_linux_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", + "attributes": { + "platform": "linux_arm64" + } + }, + "bsd_tar_windows_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%bsdtar_binary_repo", + "attributes": { + "platform": "windows_amd64" + } + }, + "bsd_tar_toolchains": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:tar_toolchain.bzl%tar_toolchains_repo", + "attributes": { + "user_repository_name": "bsd_tar" + } + }, + "zstd_darwin_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:zstd_toolchain.bzl%zstd_binary_repo", + "attributes": { + "platform": "darwin_amd64" + } + }, + "zstd_darwin_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:zstd_toolchain.bzl%zstd_binary_repo", + "attributes": { + "platform": "darwin_arm64" + } + }, + "zstd_linux_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:zstd_toolchain.bzl%zstd_binary_repo", + "attributes": { + "platform": "linux_amd64" + } + }, + "zstd_linux_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:zstd_toolchain.bzl%zstd_binary_repo", + "attributes": { + "platform": "linux_arm64" + } + }, + "zstd_toolchains": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:zstd_toolchain.bzl%zstd_toolchains_repo", + "attributes": { + "user_repository_name": "zstd" + } + }, + "expand_template_darwin_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:expand_template_toolchain.bzl%expand_template_platform_repo", + "attributes": { + "platform": "darwin_amd64" + } + }, + "expand_template_darwin_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:expand_template_toolchain.bzl%expand_template_platform_repo", + "attributes": { + "platform": "darwin_arm64" + } + }, + "expand_template_freebsd_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:expand_template_toolchain.bzl%expand_template_platform_repo", + "attributes": { + "platform": "freebsd_amd64" + } + }, + "expand_template_linux_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:expand_template_toolchain.bzl%expand_template_platform_repo", + "attributes": { + "platform": "linux_amd64" + } + }, + "expand_template_linux_arm64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:expand_template_toolchain.bzl%expand_template_platform_repo", + "attributes": { + "platform": "linux_arm64" + } + }, + "expand_template_windows_amd64": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:expand_template_toolchain.bzl%expand_template_platform_repo", + "attributes": { + "platform": "windows_amd64" + } + }, + "expand_template_toolchains": { + "repoRuleId": "@@aspect_bazel_lib+//lib/private:expand_template_toolchain.bzl%expand_template_toolchains_repo", + "attributes": { + "user_repository_name": "expand_template" + } + }, + "bats_support": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "sha256": "7815237aafeb42ddcc1b8c698fc5808026d33317d8701d5ec2396e9634e2918f", + "urls": [ + "https://github.com/bats-core/bats-support/archive/v0.3.0.tar.gz" + ], + "strip_prefix": "bats-support-0.3.0", + "build_file_content": "load(\"@aspect_bazel_lib//lib:copy_to_directory.bzl\", \"copy_to_directory\")\n\ncopy_to_directory(\n name = \"support\",\n hardlink = \"on\",\n srcs = glob([\n \"src/**\",\n \"load.bash\",\n ]),\n out = \"bats-support\",\n visibility = [\"//visibility:public\"]\n)\n" + } + }, + "bats_assert": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "sha256": "98ca3b685f8b8993e48ec057565e6e2abcc541034ed5b0e81f191505682037fd", + "urls": [ + "https://github.com/bats-core/bats-assert/archive/v2.1.0.tar.gz" + ], + "strip_prefix": "bats-assert-2.1.0", + "build_file_content": "load(\"@aspect_bazel_lib//lib:copy_to_directory.bzl\", \"copy_to_directory\")\n\ncopy_to_directory(\n name = \"assert\",\n hardlink = \"on\",\n srcs = glob([\n \"src/**\",\n \"load.bash\",\n ]),\n out = \"bats-assert\",\n visibility = [\"//visibility:public\"]\n)\n" + } + }, + "bats_file": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "sha256": "9b69043241f3af1c2d251f89b4fcafa5df3f05e97b89db18d7c9bdf5731bb27a", + "urls": [ + "https://github.com/bats-core/bats-file/archive/v0.4.0.tar.gz" + ], + "strip_prefix": "bats-file-0.4.0", + "build_file_content": "load(\"@aspect_bazel_lib//lib:copy_to_directory.bzl\", \"copy_to_directory\")\n\ncopy_to_directory(\n name = \"file\",\n hardlink = \"on\",\n srcs = glob([\n \"src/**\",\n \"load.bash\",\n ]),\n out = \"bats-file\",\n visibility = [\"//visibility:public\"]\n)\n" + } + }, + "bats_toolchains": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "sha256": "a1a9f7875aa4b6a9480ca384d5865f1ccf1b0b1faead6b47aa47d79709a5c5fd", + "urls": [ + "https://github.com/bats-core/bats-core/archive/v1.10.0.tar.gz" + ], + "strip_prefix": "bats-core-1.10.0", + "build_file_content": "load(\"@local_config_platform//:constraints.bzl\", \"HOST_CONSTRAINTS\")\nload(\"@aspect_bazel_lib//lib/private:bats_toolchain.bzl\", \"bats_toolchain\")\nload(\"@aspect_bazel_lib//lib:copy_to_directory.bzl\", \"copy_to_directory\")\n\ncopy_to_directory(\n name = \"core\",\n hardlink = \"on\",\n srcs = glob([\n \"lib/**\",\n \"libexec/**\"\n ]) + [\"bin/bats\"],\n out = \"bats-core\",\n)\n\nbats_toolchain(\n name = \"toolchain\",\n core = \":core\",\n libraries = [\"@bats_support//:support\", \"@bats_assert//:assert\", \"@bats_file//:file\"]\n)\n\ntoolchain(\n name = \"bats_toolchain\",\n exec_compatible_with = HOST_CONSTRAINTS,\n toolchain = \":toolchain\",\n toolchain_type = \"@aspect_bazel_lib//lib:bats_toolchain_type\",\n)\n" + } + } + }, + "recordedRepoMappingEntries": [ + [ + "aspect_bazel_lib+", + "aspect_bazel_lib", + "aspect_bazel_lib+" + ], + [ + "aspect_bazel_lib+", + "bazel_skylib", + "bazel_skylib+" + ], + [ + "aspect_bazel_lib+", + "bazel_tools", + "bazel_tools" + ] + ] + } + }, "@@buildifier_prebuilt+//:defs.bzl%buildifier_prebuilt_deps_extension": { "general": { "bzlTransitiveDigest": "qC/0s/MZ8q8Sf6/o/iJNMssZNgXa3CjJ7vEVbpHFQRs=", @@ -476,10 +893,199 @@ ] } }, + "@@rules_oci+//oci:extensions.bzl%oci": { + "general": { + "bzlTransitiveDigest": "vanqorv5uSAUzWdiofsrm3KqS6w5aFojwH+nl9U1nUw=", + "usagesDigest": "6UKYJNPoZGHhYW2cw0TIpkBhyshl4U5c/5tHg23Lpzs=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "python3_linux_amd64": { + "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_pull", + "attributes": { + "www_authenticate_challenges": {}, + "scheme": "https", + "registry": "index.docker.io", + "repository": "library/python", + "identifier": "sha256:e52ca5f579cc58fed41efcbb55a0ed5dccf6c7a156cba76acfb4ab42fc19dd00", + "platform": "linux/amd64", + "target_name": "python3_linux_amd64", + "bazel_tags": [] + } + }, + "python3": { + "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_alias", + "attributes": { + "target_name": "python3", + "www_authenticate_challenges": {}, + "scheme": "https", + "registry": "index.docker.io", + "repository": "library/python", + "identifier": "sha256:e52ca5f579cc58fed41efcbb55a0ed5dccf6c7a156cba76acfb4ab42fc19dd00", + "platforms": { + "@@platforms//cpu:x86_64": "@python3_linux_amd64" + }, + "bzlmod_repository": "python3", + "reproducible": true + } + }, + "oci_crane_darwin_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "darwin_amd64", + "crane_version": "v0.18.0" + } + }, + "oci_crane_darwin_arm64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "darwin_arm64", + "crane_version": "v0.18.0" + } + }, + "oci_crane_linux_arm64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "linux_arm64", + "crane_version": "v0.18.0" + } + }, + "oci_crane_linux_armv6": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "linux_armv6", + "crane_version": "v0.18.0" + } + }, + "oci_crane_linux_i386": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "linux_i386", + "crane_version": "v0.18.0" + } + }, + "oci_crane_linux_s390x": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "linux_s390x", + "crane_version": "v0.18.0" + } + }, + "oci_crane_linux_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "linux_amd64", + "crane_version": "v0.18.0" + } + }, + "oci_crane_windows_armv6": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "windows_armv6", + "crane_version": "v0.18.0" + } + }, + "oci_crane_windows_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "windows_amd64", + "crane_version": "v0.18.0" + } + }, + "oci_crane_toolchains": { + "repoRuleId": "@@rules_oci+//oci/private:toolchains_repo.bzl%toolchains_repo", + "attributes": { + "toolchain_type": "@rules_oci//oci:crane_toolchain_type", + "toolchain": "@oci_crane_{platform}//:crane_toolchain" + } + }, + "oci_regctl_darwin_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "darwin_amd64" + } + }, + "oci_regctl_darwin_arm64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "darwin_arm64" + } + }, + "oci_regctl_linux_arm64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "linux_arm64" + } + }, + "oci_regctl_linux_s390x": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "linux_s390x" + } + }, + "oci_regctl_linux_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "linux_amd64" + } + }, + "oci_regctl_windows_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "windows_amd64" + } + }, + "oci_regctl_toolchains": { + "repoRuleId": "@@rules_oci+//oci/private:toolchains_repo.bzl%toolchains_repo", + "attributes": { + "toolchain_type": "@rules_oci//oci:regctl_toolchain_type", + "toolchain": "@oci_regctl_{platform}//:regctl_toolchain" + } + } + }, + "moduleExtensionMetadata": { + "explicitRootModuleDirectDeps": [ + "python3", + "python3_linux_amd64" + ], + "explicitRootModuleDirectDevDeps": [], + "useAllRepos": "NO", + "reproducible": false + }, + "recordedRepoMappingEntries": [ + [ + "aspect_bazel_lib+", + "bazel_tools", + "bazel_tools" + ], + [ + "bazel_features+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_oci+", + "aspect_bazel_lib", + "aspect_bazel_lib+" + ], + [ + "rules_oci+", + "bazel_features", + "bazel_features+" + ], + [ + "rules_oci+", + "bazel_skylib", + "bazel_skylib+" + ] + ] + } + }, "@@rules_python+//python/extensions:pip.bzl%pip": { "general": { "bzlTransitiveDigest": "Na3CSqXZ/0ryNiFlQYazko5xszx5rVFuJu6gjTA5duQ=", - "usagesDigest": "lZB80AepWucblvwrn0cps30+cGAb+iflR0IAjxse8Ns=", + "usagesDigest": "8C4xkdewwejlDg2Hc75ZGzeBesJsgAtTnl7Ent7boIg=", "recordedFileInputs": { "@@//requirements.txt": "93db0fe77ec706524749f1ff6f1c6d8dd314e90a6e1041d6d51520a0abab48c6", "@@protobuf+//python/requirements.txt": "983be60d3cec4b319dcab6d48aeb3f5b2f7c3350f26b3a9e97486c37967c73c5", From 2e312e1e16086373adad1e479f79b90c28a351b6 Mon Sep 17 00:00:00 2001 From: gpupo Date: Tue, 1 Apr 2025 21:04:32 -0300 Subject: [PATCH 08/13] Update python image version --- MODULE.bazel | 6 +++--- MODULE.bazel.lock | 6 +++--- README.md | 6 ++++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 8ee0da5..9981597 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -50,8 +50,8 @@ oci = use_extension("@rules_oci//oci:extensions.bzl", "oci") # INFO: https://hub.docker.com/_/python oci.pull( name = "python3", - digest = "sha256:e52ca5f579cc58fed41efcbb55a0ed5dccf6c7a156cba76acfb4ab42fc19dd00", - image = "python:3.9-slim", + digest = "sha256:a866731a6b71c4a194a845d86e06568725e430ed21821d0c52e4efb385cf6c6f", + image = "python:3.12.9-slim", platforms = ["linux/amd64"], ) -use_repo(oci, "python3") +use_repo(oci, "python3", "python3_linux_amd64") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 4014eb1..d556c91 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -896,7 +896,7 @@ "@@rules_oci+//oci:extensions.bzl%oci": { "general": { "bzlTransitiveDigest": "vanqorv5uSAUzWdiofsrm3KqS6w5aFojwH+nl9U1nUw=", - "usagesDigest": "6UKYJNPoZGHhYW2cw0TIpkBhyshl4U5c/5tHg23Lpzs=", + "usagesDigest": "pcBjRehAXosWF6+v3Psb2Cw+dWQah1f6cXzj1DogbjE=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -908,7 +908,7 @@ "scheme": "https", "registry": "index.docker.io", "repository": "library/python", - "identifier": "sha256:e52ca5f579cc58fed41efcbb55a0ed5dccf6c7a156cba76acfb4ab42fc19dd00", + "identifier": "sha256:a866731a6b71c4a194a845d86e06568725e430ed21821d0c52e4efb385cf6c6f", "platform": "linux/amd64", "target_name": "python3_linux_amd64", "bazel_tags": [] @@ -922,7 +922,7 @@ "scheme": "https", "registry": "index.docker.io", "repository": "library/python", - "identifier": "sha256:e52ca5f579cc58fed41efcbb55a0ed5dccf6c7a156cba76acfb4ab42fc19dd00", + "identifier": "sha256:a866731a6b71c4a194a845d86e06568725e430ed21821d0c52e4efb385cf6c6f", "platforms": { "@@platforms//cpu:x86_64": "@python3_linux_amd64" }, diff --git a/README.md b/README.md index f279ef6..2ad7292 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Para rodar os testes do projeto, utilize: bazel test //... ``` -## DAG Apache Airflow +## Simple DAG Apache Airflow Rodando os testes unitarios @@ -56,7 +56,9 @@ Rodando a adaptacao para ambiente local bazel run //dag/simple_dag:dag_dev -TODO: Adicionar DockerOperator +## DockerOperator + + bazel build //dag/docker-operator-dag:python3_image ## Requisitos From b13d691307eb37ab41f76cb1fb3c3dc699639e76 Mon Sep 17 00:00:00 2001 From: gpupo Date: Tue, 1 Apr 2025 21:30:19 -0300 Subject: [PATCH 09/13] add docker operator build file --- dag/docker-operator-dag/BUILD.bazel | 35 +++++++++++++++++++++++++++++ dag/docker-operator-dag/task.py | 1 + 2 files changed, 36 insertions(+) create mode 100644 dag/docker-operator-dag/BUILD.bazel create mode 100644 dag/docker-operator-dag/task.py diff --git a/dag/docker-operator-dag/BUILD.bazel b/dag/docker-operator-dag/BUILD.bazel new file mode 100644 index 0000000..7b6fdf9 --- /dev/null +++ b/dag/docker-operator-dag/BUILD.bazel @@ -0,0 +1,35 @@ +load("@pypi//:requirements.bzl", "requirement") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load") +load("@rules_pkg//pkg:tar.bzl", "pkg_tar") +load("@rules_python//python:defs.bzl", "py_binary", "py_library") + +py_binary( + name = "task", + srcs = ["task.py"], + main = "task.py", + deps = [], +) + +# INFO: Package the Python binary into a tar archive +pkg_tar( + name = "task_tar", + srcs = [":task"], + include_runfiles = True, # Python binaries + package_dir = "/app", # Where files will be placed in container +) + +# INFO: Cria a imagem +oci_image( + name = "image", + base = "@python3", + cmd = ["python3"], + entrypoint = ["/app/task"], # Path binary in container + tars = [":task_tar"], # packaged application +) + +# INFO: Registra a imagem no ambiente +oci_load( + name = "image_load", + image = ":image", + repo_tags = ["docker-operator-dag:latest"], +) diff --git a/dag/docker-operator-dag/task.py b/dag/docker-operator-dag/task.py new file mode 100644 index 0000000..44159b3 --- /dev/null +++ b/dag/docker-operator-dag/task.py @@ -0,0 +1 @@ +print("Hello world") From 7548cbd838d362584175a2dfa7d14a9693c11ff1 Mon Sep 17 00:00:00 2001 From: gpupo Date: Wed, 2 Apr 2025 07:03:59 -0300 Subject: [PATCH 10/13] add docker operator dag; dependencie and local dev file --- dag/docker-operator-dag/dag.py | 138 +++++++++++++++++++++++++++ dag/docker-operator-dag/local_dev.py | 32 +++++++ dag/docker-operator-dag/task.py | 2 +- pyproject.toml | 1 + 4 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 dag/docker-operator-dag/dag.py create mode 100644 dag/docker-operator-dag/local_dev.py diff --git a/dag/docker-operator-dag/dag.py b/dag/docker-operator-dag/dag.py new file mode 100644 index 0000000..d7156ac --- /dev/null +++ b/dag/docker-operator-dag/dag.py @@ -0,0 +1,138 @@ +from datetime import datetime + +from airflow import DAG +from airflow.providers.docker.operators.docker import DockerOperator + +with DAG( + "docker_operator_example", + schedule=None, + start_date=datetime(2023, 1, 1), +) as dag: + """ + Execute a command inside a docker container. + + By default, a temporary directory is + created on the host and mounted into a container to allow storing files + that together exceed the default disk size of 10GB in a container. + In this case The path to the mounted directory can be accessed + via the environment variable ``AIRFLOW_TMP_DIR``. + + If the volume cannot be mounted, warning is printed and an attempt is made to execute the docker + command without the temporary folder mounted. This is to make it works by default with remote docker + engine or when you run docker-in-docker solution and temporary directory is not shared with the + docker engine. Warning is printed in logs in this case. + + If you know you run DockerOperator with remote engine or via docker-in-docker + you should set ``mount_tmp_dir`` parameter to False. In this case, you can still use + ``mounts`` parameter to mount already existing named volumes in your Docker Engine + to achieve similar capability where you can store files exceeding default disk size + of the container, + + If a login to a private registry is required prior to pulling the image, a + Docker connection needs to be configured in Airflow and the connection ID + be provided with the parameter ``docker_conn_id``. + + :param image: Docker image from which to create the container. + If image tag is omitted, "latest" will be used. (templated) + :param api_version: Remote API version. Set to ``auto`` to automatically + detect the server's version. + :param command: Command to be run in the container. (templated) + :param container_name: Name of the container. Optional (templated) + :param cpus: Number of CPUs to assign to the container. + This value gets multiplied with 1024. See + https://docs.docker.com/engine/reference/run/#cpu-share-constraint + :param docker_url: URL or list of URLs of the host(s) running the docker daemon. + Default is the value of the ``DOCKER_HOST`` environment variable or unix://var/run/docker.sock + if it is unset. + :param environment: Environment variables to set in the container. (templated) + :param private_environment: Private environment variables to set in the container. + These are not templated, and hidden from the website. + :param env_file: Relative path to the ``.env`` file with environment variables to set in the container. + Overridden by variables in the environment parameter. (templated) + :param force_pull: Pull the docker image on every run. Default is False. + :param mem_limit: Maximum amount of memory the container can use. + Either a float value, which represents the limit in bytes, + or a string like ``128m`` or ``1g``. + :param host_tmp_dir: Specify the location of the temporary directory on the host which will + be mapped to tmp_dir. If not provided defaults to using the standard system temp directory. + :param network_mode: Network mode for the container. It can be one of the following: + + - ``"bridge"``: Create new network stack for the container with default docker bridge network + - ``"none"``: No networking for this container + - ``"container:"``: Use the network stack of another container specified via + - ``"host"``: Use the host network stack. Incompatible with `port_bindings` + - ``"|"``: Connects the container to user created network + (using ``docker network create`` command) + :param tls_ca_cert: Path to a PEM-encoded certificate authority + to secure the docker connection. + :param tls_client_cert: Path to the PEM-encoded certificate + used to authenticate docker client. + :param tls_client_key: Path to the PEM-encoded key used to authenticate docker client. + :param tls_verify: Set ``True`` to verify the validity of the provided certificate. + :param tls_hostname: Hostname to match against + the docker server certificate or False to disable the check. + :param tls_ssl_version: Version of SSL to use when communicating with docker daemon. + :param mount_tmp_dir: Specify whether the temporary directory should be bind-mounted + from the host to the container. Defaults to True + :param tmp_dir: Mount point inside the container to + a temporary directory created on the host by the operator. + The path is also made available via the environment variable + ``AIRFLOW_TMP_DIR`` inside the container. + :param user: Default user inside the docker container. + :param mounts: List of volumes to mount into the container. Each item should + be a :py:class:`docker.types.Mount` instance. + :param entrypoint: Overwrite the default ENTRYPOINT of the image + :param working_dir: Working directory to + set on the container (equivalent to the -w switch the docker client) + :param xcom_all: Push all the stdout or just the last line. + The default is False (last line). + :param docker_conn_id: The :ref:`Docker connection id ` + :param dns: Docker custom DNS servers + :param dns_search: Docker custom DNS search domain + :param auto_remove: Enable removal of the container when the container's process exits. Possible values: + + - ``never``: (default) do not remove container + - ``success``: remove on success + - ``force``: always remove container + :param shm_size: Size of ``/dev/shm`` in bytes. The size must be + greater than 0. If omitted uses system default. + :param tty: Allocate pseudo-TTY to the container + This needs to be set see logs of the Docker container. + :param hostname: Optional hostname for the container. + :param privileged: Give extended privileges to this container. + :param cap_add: Include container capabilities + :param extra_hosts: Additional hostnames to resolve inside the container, + as a mapping of hostname to IP address. + :param retrieve_output: Should this docker image consistently attempt to pull from and output + file before manually shutting down the image. Useful for cases where users want a pickle serialized + output that is not posted to logs + :param retrieve_output_path: path for output file that will be retrieved and passed to xcom + :param timeout: Timeout for API calls, in seconds. Default is 60 seconds. + :param device_requests: Expose host resources such as GPUs to the container. + :param log_opts_max_size: The maximum size of the log before it is rolled. + A positive integer plus a modifier representing the unit of measure (k, m, or g). + Eg: 10m or 1g Defaults to -1 (unlimited). + :param log_opts_max_file: The maximum number of log files that can be present. + If rolling the logs creates excess files, the oldest file is removed. + Only effective when max-size is also set. A positive integer. Defaults to 1. + :param ipc_mode: Set the IPC mode for the container. + :param skip_on_exit_code: If task exits with this exit code, leave the task + in ``skipped`` state (default: None). If set to ``None``, any non-zero + exit code will be treated as a failure. + :param port_bindings: Publish a container's port(s) to the host. It is a + dictionary of value where the key indicates the port to open inside the container + and value indicates the host port that binds to the container port. + Incompatible with ``"host"`` in ``network_mode``. + :param ulimits: List of ulimit options to set for the container. Each item should + be a :py:class:`docker.types.Ulimit` instance. + :param labels: A dictionary of name-value labels (e.g. ``{"label1": "value1", "label2": "value2"}``) + or a list of names of labels to set with empty values (e.g. ``["label1", "label2"]``) + """ + run_task = DockerOperator( + task_id="run_in_docker", + image="docker-operator-dag:latest", + api_version="auto", + # auto_remove=True, + docker_url="unix:///var/run/docker.sock", # Ou "tcp://localhost:2375" + network_mode="bridge", + ) diff --git a/dag/docker-operator-dag/local_dev.py b/dag/docker-operator-dag/local_dev.py new file mode 100644 index 0000000..d742aec --- /dev/null +++ b/dag/docker-operator-dag/local_dev.py @@ -0,0 +1,32 @@ +import os +import sys +from datetime import datetime + +# Add Airflow to Python path +sys.path.append(os.path.dirname(os.path.abspath(__file__))) + +# Import the DAG +from dag import dag + + +def local_run(): + """ + Run the DAG locally for testing and development + """ + # Get the DAG object + print("Running DAG locally...") + + # Run each task sequentially + for task in dag.tasks: + print(f"Executing task: {task.task_id}") + try: + task.execute( + context={"dag": dag, "task": task, "execution_date": datetime.now()} + ) + except Exception as e: + print(f"Error executing {task.task_id}: {e}") + raise + + +if __name__ == "__main__": + local_run() diff --git a/dag/docker-operator-dag/task.py b/dag/docker-operator-dag/task.py index 44159b3..3c3c474 100644 --- a/dag/docker-operator-dag/task.py +++ b/dag/docker-operator-dag/task.py @@ -1 +1 @@ -print("Hello world") +print("Hello World from the Task!") diff --git a/pyproject.toml b/pyproject.toml index a9e370e..d37594a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,7 @@ requires-python = ">=3.12" dependencies = [ "apache-airflow>=2.10.5", "apache-airflow-providers-common-sql>=1.21.0", + "apache-airflow-providers-docker>=4.3.0", "apache-airflow-providers-postgres>=6.0.0", "apache-airflow-providers-sqlite>=4.0.0", "pytest>=8.3.4", From be5c1fc57dba5acdb331f40ca9612eb2c5f8c1ce Mon Sep 17 00:00:00 2001 From: gpupo Date: Wed, 2 Apr 2025 07:04:23 -0300 Subject: [PATCH 11/13] update uv lock --- uv.lock | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/uv.lock b/uv.lock index a36b2f6..924f8bb 100644 --- a/uv.lock +++ b/uv.lock @@ -226,6 +226,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2b/9f/5db4e5d34fde80def8621f8356002ce6ef806164b693a7e397d6ae6e6266/apache_airflow_providers_common_sql-1.21.0-py3-none-any.whl", hash = "sha256:87d26eab25fb09e0b04e14f563dfc3a365fc86e2452211e075229dcb8ce142d2", size = 49879 }, ] +[[package]] +name = "apache-airflow-providers-docker" +version = "4.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "apache-airflow" }, + { name = "docker" }, + { name = "python-dotenv" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/80/4f/cfc43cf9452d7430ab1ef51f8e46095d77c0ad7b4043dd99b72992e54144/apache_airflow_providers_docker-4.3.0.tar.gz", hash = "sha256:992c47e2151f3de8ef2b7a2beb4e75fbab592fb07cfe5741748dd1869d5e4de8", size = 22592 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/2e/cf7e056190b37d7355ee481f72f7f49ed4e732badf39e3b32a49c10ecc96/apache_airflow_providers_docker-4.3.0-py3-none-any.whl", hash = "sha256:e63f79ddfcee4e9af7e5dc2d08af12f6035c7420f706d045904578d1a69e58cf", size = 31704 }, +] + [[package]] name = "apache-airflow-providers-fab" version = "1.5.2" @@ -409,6 +423,7 @@ source = { virtual = "." } dependencies = [ { name = "apache-airflow" }, { name = "apache-airflow-providers-common-sql" }, + { name = "apache-airflow-providers-docker" }, { name = "apache-airflow-providers-postgres" }, { name = "apache-airflow-providers-sqlite" }, { name = "pytest" }, @@ -419,6 +434,7 @@ dependencies = [ requires-dist = [ { name = "apache-airflow", specifier = ">=2.10.5" }, { name = "apache-airflow-providers-common-sql", specifier = ">=1.21.0" }, + { name = "apache-airflow-providers-docker", specifier = ">=4.3.0" }, { name = "apache-airflow-providers-postgres", specifier = ">=6.0.0" }, { name = "apache-airflow-providers-sqlite", specifier = ">=4.0.0" }, { name = "pytest", specifier = ">=8.3.4" }, @@ -687,6 +703,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/68/1b/e0a87d256e40e8c888847551b20a017a6b98139178505dc7ffb96f04e954/dnspython-2.7.0-py3-none-any.whl", hash = "sha256:b4c34b7d10b51bcc3a5071e7b8dee77939f1e878477eeecc965e9835f63c6c86", size = 313632 }, ] +[[package]] +name = "docker" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "requests" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/91/9b/4a2ea29aeba62471211598dac5d96825bb49348fa07e906ea930394a83ce/docker-7.1.0.tar.gz", hash = "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c", size = 117834 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/26/57c6fb270950d476074c087527a558ccb6f4436657314bfb6cdf484114c4/docker-7.1.0-py3-none-any.whl", hash = "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0", size = 147774 }, +] + [[package]] name = "email-validator" version = "2.2.0" @@ -1724,6 +1754,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892 }, ] +[[package]] +name = "python-dotenv" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/88/2c/7bb1416c5620485aa793f2de31d3df393d3686aa8a8506d11e10e13c5baf/python_dotenv-1.1.0.tar.gz", hash = "sha256:41f90bc6f5f177fb41f53e87666db362025010eb28f60a01c9143bfa33a2b2d5", size = 39920 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/18/98a99ad95133c6a6e2005fe89faedf294a748bd5dc803008059409ac9b1e/python_dotenv-1.1.0-py3-none-any.whl", hash = "sha256:d7c01d9e2293916c18baf562d95698754b0dbbb5e74d457c45d4f6561fb9d55d", size = 20256 }, +] + [[package]] name = "python-nvd3" version = "0.16.0" @@ -1755,6 +1794,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/eb/38/ac33370d784287baa1c3d538978b5e2ea064d4c1b93ffbd12826c190dd10/pytz-2025.1-py2.py3-none-any.whl", hash = "sha256:89dd22dca55b46eac6eda23b2d72721bf1bdfef212645d81513ef5d03038de57", size = 507930 }, ] +[[package]] +name = "pywin32" +version = "310" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/ec/4fdbe47932f671d6e348474ea35ed94227fb5df56a7c30cbbb42cd396ed0/pywin32-310-cp312-cp312-win32.whl", hash = "sha256:8a75a5cc3893e83a108c05d82198880704c44bbaee4d06e442e471d3c9ea4f3d", size = 8796239 }, + { url = "https://files.pythonhosted.org/packages/e3/e5/b0627f8bb84e06991bea89ad8153a9e50ace40b2e1195d68e9dff6b03d0f/pywin32-310-cp312-cp312-win_amd64.whl", hash = "sha256:bf5c397c9a9a19a6f62f3fb821fbf36cac08f03770056711f765ec1503972060", size = 9503839 }, + { url = "https://files.pythonhosted.org/packages/1f/32/9ccf53748df72301a89713936645a664ec001abd35ecc8578beda593d37d/pywin32-310-cp312-cp312-win_arm64.whl", hash = "sha256:2349cc906eae872d0663d4d6290d13b90621eaf78964bb1578632ff20e152966", size = 8459470 }, + { url = "https://files.pythonhosted.org/packages/1c/09/9c1b978ffc4ae53999e89c19c77ba882d9fce476729f23ef55211ea1c034/pywin32-310-cp313-cp313-win32.whl", hash = "sha256:5d241a659c496ada3253cd01cfaa779b048e90ce4b2b38cd44168ad555ce74ab", size = 8794384 }, + { url = "https://files.pythonhosted.org/packages/45/3c/b4640f740ffebadd5d34df35fecba0e1cfef8fde9f3e594df91c28ad9b50/pywin32-310-cp313-cp313-win_amd64.whl", hash = "sha256:667827eb3a90208ddbdcc9e860c81bde63a135710e21e4cb3348968e4bd5249e", size = 9503039 }, + { url = "https://files.pythonhosted.org/packages/b4/f4/f785020090fb050e7fb6d34b780f2231f302609dc964672f72bfaeb59a28/pywin32-310-cp313-cp313-win_arm64.whl", hash = "sha256:e308f831de771482b7cf692a1f308f8fca701b2d8f9dde6cc440c7da17e47b33", size = 8458152 }, +] + [[package]] name = "pyyaml" version = "6.0.2" From c81fb3bf282603487dea5ad835a636f252ffc52a Mon Sep 17 00:00:00 2001 From: gpupo Date: Wed, 2 Apr 2025 07:36:15 -0300 Subject: [PATCH 12/13] add docker operator build; clean local dev --- dag/docker-operator-dag/BUILD.bazel | 8 ++++---- dag/docker-operator-dag/local_dev.py | 6 ------ 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/dag/docker-operator-dag/BUILD.bazel b/dag/docker-operator-dag/BUILD.bazel index 7b6fdf9..e5045d4 100644 --- a/dag/docker-operator-dag/BUILD.bazel +++ b/dag/docker-operator-dag/BUILD.bazel @@ -14,8 +14,8 @@ py_binary( pkg_tar( name = "task_tar", srcs = [":task"], - include_runfiles = True, # Python binaries - package_dir = "/app", # Where files will be placed in container + include_runfiles = True, + package_dir = "/app", ) # INFO: Cria a imagem @@ -23,8 +23,8 @@ oci_image( name = "image", base = "@python3", cmd = ["python3"], - entrypoint = ["/app/task"], # Path binary in container - tars = [":task_tar"], # packaged application + entrypoint = ["/app/task"], + tars = [":task_tar"], ) # INFO: Registra a imagem no ambiente diff --git a/dag/docker-operator-dag/local_dev.py b/dag/docker-operator-dag/local_dev.py index d742aec..560aad5 100644 --- a/dag/docker-operator-dag/local_dev.py +++ b/dag/docker-operator-dag/local_dev.py @@ -1,10 +1,5 @@ -import os -import sys from datetime import datetime -# Add Airflow to Python path -sys.path.append(os.path.dirname(os.path.abspath(__file__))) - # Import the DAG from dag import dag @@ -13,7 +8,6 @@ def local_run(): """ Run the DAG locally for testing and development """ - # Get the DAG object print("Running DAG locally...") # Run each task sequentially From 07564595f7e5e09a165196284cfe1cb49a9dbc6e Mon Sep 17 00:00:00 2001 From: gpupo Date: Wed, 2 Apr 2025 07:36:22 -0300 Subject: [PATCH 13/13] dag doc --- dag/docker-operator-dag/README.md | 133 ++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 dag/docker-operator-dag/README.md diff --git a/dag/docker-operator-dag/README.md b/dag/docker-operator-dag/README.md new file mode 100644 index 0000000..f60720b --- /dev/null +++ b/dag/docker-operator-dag/README.md @@ -0,0 +1,133 @@ +# Docker Operator DAG with Bazel + +This directory contains an Apache Airflow DAG that demonstrates how to use the `DockerOperator` with Bazel for containerized task execution. + +## Overview + +The implementation: + +1. Packages a Python task into a Docker container using Bazel +2. Creates an Airflow DAG that runs the task in a container +3. Provides local development and testing capabilities + +## Structure + +``` +docker-operator-dag/ +├── BUILD.bazel # Bazel build rules +├── dag.py # Airflow DAG definition +├── Dockerfile # Docker configuration (alternative to Bazel) +├── local_dev.py # Local execution script +├── task.py # Python task to run in container +└── README.md # This documentation +``` + +## How It Works + +### 1. Building the Container Image + +The Bazel build system packages the Python task into a container: + +```bash +# Build the container image +bazel build //dag/docker-operator-dag:image + +# Load the image into Docker +bazel run //dag/docker-operator-dag:image_load +``` + +### 2. Running the DAG + +The DAG (`dag.py`) uses `DockerOperator` to execute the task in the container: + +```python +from airflow import DAG +from airflow.providers.docker.operators.docker import DockerOperator + +with DAG( + 'docker_operator_example', + default_args=default_args, + schedule_interval=None +) as dag: + + run_task = DockerOperator( + task_id='run_in_docker', + image='docker-operator-dag:latest', # Matches repo_tags in BUILD.bazel + api_version='auto', + auto_remove=True, + command='python /app/task.py' + ) +``` + +### 3. Local Development + +Test the DAG locally without Airflow: + +```bash +# Execute the DAG tasks locally +bazel run //dag/docker-operator-dag:local_dev +``` + +## Key Components + +### `task.py` + +The Python script that runs inside the container. Modify this for your specific task logic. + +### `BUILD.bazel` + +Defines the build pipeline: + +- Creates a Python binary from `task.py` +- Packages it into a tarball +- Builds an OCI container image +- Provides a load rule for local Docker + +### `local_dev.py` + +Provides a local execution environment that: + +- Simulates Airflow's task execution +- Runs each task sequentially +- Provides basic logging + +## Development Workflow + +1. Edit `task.py` with your task logic +2. Build and load the container: + ```bash + bazel run //dag/docker-operator-dag:image_load + ``` +3. Test locally: + ```bash + bazel run //dag/docker-operator-dag:local_dev + ``` +4. Deploy to Airflow + +## Customizing + +1. To add Python dependencies: + + - Add to `requirements.txt` in project root + - Include in `deps` in `BUILD.bazel` + +2. To change the base image: + - Update the `base` parameter in `oci_image` rule + - Or modify the `Dockerfile` for custom builds + +## Troubleshooting + +If the container fails to run: + +1. Verify the image loaded correctly: + ```bash + docker images | grep docker-operator-dag + ``` +2. Test the container manually: + ```bash + docker run --rm docker-operator-dag:latest + ``` +3. Check Bazel outputs: + ```bash + bazel query //dag/docker-operator-dag:all --output=build + ```