From 5654165d0cbdbc62ff47c8cac4b56ef682bfe57b Mon Sep 17 00:00:00 2001 From: brunofabio <31631841+brunofabio@users.noreply.github.com> Date: Tue, 7 Apr 2020 00:59:49 -0500 Subject: [PATCH 1/8] Set up CI with Azure Pipelines [skip ci] --- .pipelines/diabetes_regression-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.pipelines/diabetes_regression-ci.yml b/.pipelines/diabetes_regression-ci.yml index 56258d50..c53bca79 100644 --- a/.pipelines/diabetes_regression-ci.yml +++ b/.pipelines/diabetes_regression-ci.yml @@ -1,4 +1,7 @@ # Continuous Integration (CI) pipeline that orchestrates the training, evaluation, registration, deployment, and testing of the diabetes_regression model. +- task: DotNetCoreCLI@2 + inputs: + command: 'build' resources: containers: From cf1e6bef57d800597f1311001911e69cccb4809f Mon Sep 17 00:00:00 2001 From: brunofabio <31631841+brunofabio@users.noreply.github.com> Date: Tue, 7 Apr 2020 01:02:23 -0500 Subject: [PATCH 2/8] Set up CI with Azure Pipelines [skip ci] --- .pipelines/diabetes_regression-ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.pipelines/diabetes_regression-ci.yml b/.pipelines/diabetes_regression-ci.yml index c53bca79..56258d50 100644 --- a/.pipelines/diabetes_regression-ci.yml +++ b/.pipelines/diabetes_regression-ci.yml @@ -1,7 +1,4 @@ # Continuous Integration (CI) pipeline that orchestrates the training, evaluation, registration, deployment, and testing of the diabetes_regression model. -- task: DotNetCoreCLI@2 - inputs: - command: 'build' resources: containers: From 30238bbd113a9159ab8cc65ac2503366a715bcdf Mon Sep 17 00:00:00 2001 From: BRUNO Date: Tue, 7 Apr 2020 12:52:46 -0500 Subject: [PATCH 3/8] cambio load_sample_data sin parametro file_name --- ml_service/pipelines/load_sample_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ml_service/pipelines/load_sample_data.py b/ml_service/pipelines/load_sample_data.py index cad56568..717fc7ab 100644 --- a/ml_service/pipelines/load_sample_data.py +++ b/ml_service/pipelines/load_sample_data.py @@ -5,7 +5,7 @@ # Loads the diabetes sample data from sklearn and produces a csv file that can # be used by the build/train pipeline script. -def create_sample_data_csv(file_name): +def create_sample_data_csv(): sample_data = load_diabetes() df = pd.DataFrame( data=sample_data.data, From 39c3bf04cc07218a9f812ec806e52de9236e8489 Mon Sep 17 00:00:00 2001 From: BRUNO Date: Tue, 21 Apr 2020 01:55:56 -0500 Subject: [PATCH 4/8] cambio en train, nuevo comentario --- diabetes_regression/training/train.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/diabetes_regression/training/train.py b/diabetes_regression/training/train.py index 22258042..acd1886d 100644 --- a/diabetes_regression/training/train.py +++ b/diabetes_regression/training/train.py @@ -24,6 +24,8 @@ POSSIBILITY OF SUCH DAMAGE. """ +# Importación de Librerias + import os import pandas as pd from sklearn.linear_model import Ridge From 69e5fb2d45bc7b74019e344b96afb340c65e80c9 Mon Sep 17 00:00:00 2001 From: BRUNO Date: Thu, 23 Apr 2020 19:50:09 -0500 Subject: [PATCH 5/8] datafactory --- data_factory/temp.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 data_factory/temp.txt diff --git a/data_factory/temp.txt b/data_factory/temp.txt new file mode 100644 index 00000000..e69de29b From 6aabf27b8bf36a89f751ae3a295f5672bdb6ecd5 Mon Sep 17 00:00:00 2001 From: brunofabio <31631841+brunofabio@users.noreply.github.com> Date: Thu, 23 Apr 2020 20:17:21 -0500 Subject: [PATCH 6/8] Adding linkedService: inputblob --- data_factory/linkedService/inputblob.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 data_factory/linkedService/inputblob.json diff --git a/data_factory/linkedService/inputblob.json b/data_factory/linkedService/inputblob.json new file mode 100644 index 00000000..7c92d260 --- /dev/null +++ b/data_factory/linkedService/inputblob.json @@ -0,0 +1,12 @@ +{ + "name": "inputblob", + "type": "Microsoft.DataFactory/factories/linkedservices", + "properties": { + "annotations": [], + "type": "AzureBlobStorage", + "typeProperties": { + "connectionString": "DefaultEndpointsProtocol=https;AccountName=brumlopsamlsa;EndpointSuffix=core.windows.net;", + "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkRGLU1MT1BTX2ViYWQwYzRhLTZlOTktNDk1OC1iMTE0LTkzOGVkNGY5ZDliMSINCn0=" + } + } +} \ No newline at end of file From bf6de2c6a2e2f44d8fe574f9400077086b79a30c Mon Sep 17 00:00:00 2001 From: brunofabio <31631841+brunofabio@users.noreply.github.com> Date: Thu, 23 Apr 2020 20:36:03 -0500 Subject: [PATCH 7/8] Adding pipeline: data to mlops Adding dataset: diabetesCSV --- data_factory/dataset/diabetesCSV.json | 68 ++++++++++++++++++++++++ data_factory/pipeline/data to mlops.json | 57 ++++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 data_factory/dataset/diabetesCSV.json create mode 100644 data_factory/pipeline/data to mlops.json diff --git a/data_factory/dataset/diabetesCSV.json b/data_factory/dataset/diabetesCSV.json new file mode 100644 index 00000000..5e511221 --- /dev/null +++ b/data_factory/dataset/diabetesCSV.json @@ -0,0 +1,68 @@ +{ + "name": "diabetesCSV", + "properties": { + "linkedServiceName": { + "referenceName": "inputblob", + "type": "LinkedServiceReference" + }, + "annotations": [], + "type": "DelimitedText", + "typeProperties": { + "location": { + "type": "AzureBlobStorageLocation", + "fileName": "diabetes.csv", + "container": "datainput" + }, + "columnDelimiter": ",", + "escapeChar": "\\", + "firstRowAsHeader": true, + "quoteChar": "\"" + }, + "schema": [ + { + "name": "AGE", + "type": "String" + }, + { + "name": "SEX", + "type": "String" + }, + { + "name": "BMI", + "type": "String" + }, + { + "name": "BP", + "type": "String" + }, + { + "name": "S1", + "type": "String" + }, + { + "name": "S2", + "type": "String" + }, + { + "name": "S3", + "type": "String" + }, + { + "name": "S4", + "type": "String" + }, + { + "name": "S5", + "type": "String" + }, + { + "name": "S6", + "type": "String" + }, + { + "name": "Y", + "type": "String" + } + ] + } +} \ No newline at end of file diff --git a/data_factory/pipeline/data to mlops.json b/data_factory/pipeline/data to mlops.json new file mode 100644 index 00000000..6916b655 --- /dev/null +++ b/data_factory/pipeline/data to mlops.json @@ -0,0 +1,57 @@ +{ + "name": "data to mlops", + "properties": { + "activities": [ + { + "name": "data to ml model", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "DelimitedTextSource", + "storeSettings": { + "type": "AzureBlobStorageReadSettings", + "recursive": true + }, + "formatSettings": { + "type": "DelimitedTextReadSettings" + } + }, + "sink": { + "type": "DelimitedTextSink", + "storeSettings": { + "type": "AzureBlobStorageWriteSettings" + }, + "formatSettings": { + "type": "DelimitedTextWriteSettings", + "quoteAllText": true, + "fileExtension": ".txt" + } + }, + "enableStaging": false + }, + "inputs": [ + { + "referenceName": "diabetesCSV", + "type": "DatasetReference" + } + ], + "outputs": [ + { + "referenceName": "diabetesCSV", + "type": "DatasetReference" + } + ] + } + ], + "annotations": [] + } +} \ No newline at end of file From 86e63ce6e8313c135a4fb509263f1429e2f45314 Mon Sep 17 00:00:00 2001 From: brunofabio <31631841+brunofabio@users.noreply.github.com> Date: Thu, 23 Apr 2020 20:38:52 -0500 Subject: [PATCH 8/8] Adding templates/data_to_mlops: data_to_mlops Adding templates/data_to_mlops: manifest --- .../data_to_mlops/data_to_mlops.json | 151 ++++++++++++++++++ .../templates/data_to_mlops/manifest.json | 21 +++ 2 files changed, 172 insertions(+) create mode 100644 data_factory/templates/data_to_mlops/data_to_mlops.json create mode 100644 data_factory/templates/data_to_mlops/manifest.json diff --git a/data_factory/templates/data_to_mlops/data_to_mlops.json b/data_factory/templates/data_to_mlops/data_to_mlops.json new file mode 100644 index 00000000..5f9f81fa --- /dev/null +++ b/data_factory/templates/data_to_mlops/data_to_mlops.json @@ -0,0 +1,151 @@ +{ + "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "factoryName": { + "type": "string", + "metadata": "Data Factory name" + }, + "inputblob": { + "type": "string" + } + }, + "variables": { + "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" + }, + "resources": [ + { + "name": "[concat(parameters('factoryName'), '/data to mlops')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "data to ml model", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "DelimitedTextSource", + "storeSettings": { + "type": "AzureBlobStorageReadSettings", + "recursive": true + }, + "formatSettings": { + "type": "DelimitedTextReadSettings" + } + }, + "sink": { + "type": "DelimitedTextSink", + "storeSettings": { + "type": "AzureBlobStorageWriteSettings" + }, + "formatSettings": { + "type": "DelimitedTextWriteSettings", + "quoteAllText": true, + "fileExtension": ".txt" + } + }, + "enableStaging": false + }, + "inputs": [ + { + "referenceName": "diabetesCSV", + "type": "DatasetReference" + } + ], + "outputs": [ + { + "referenceName": "diabetesCSV", + "type": "DatasetReference" + } + ] + } + ], + "annotations": [] + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/datasets/diabetesCSV')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/diabetesCSV')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "[parameters('inputblob')]", + "type": "LinkedServiceReference" + }, + "annotations": [], + "type": "DelimitedText", + "typeProperties": { + "location": { + "type": "AzureBlobStorageLocation", + "fileName": "diabetes.csv", + "container": "datainput" + }, + "columnDelimiter": ",", + "escapeChar": "\\", + "firstRowAsHeader": true, + "quoteChar": "\"" + }, + "schema": [ + { + "name": "AGE", + "type": "String" + }, + { + "name": "SEX", + "type": "String" + }, + { + "name": "BMI", + "type": "String" + }, + { + "name": "BP", + "type": "String" + }, + { + "name": "S1", + "type": "String" + }, + { + "name": "S2", + "type": "String" + }, + { + "name": "S3", + "type": "String" + }, + { + "name": "S4", + "type": "String" + }, + { + "name": "S5", + "type": "String" + }, + { + "name": "S6", + "type": "String" + }, + { + "name": "Y", + "type": "String" + } + ] + }, + "dependsOn": [] + } + ] +} \ No newline at end of file diff --git a/data_factory/templates/data_to_mlops/manifest.json b/data_factory/templates/data_to_mlops/manifest.json new file mode 100644 index 00000000..b845b734 --- /dev/null +++ b/data_factory/templates/data_to_mlops/manifest.json @@ -0,0 +1,21 @@ +{ + "name": "data_to_mlops", + "image": "Copy datadata to ml model", + "icons": [ + "Copy", + "DelimitedText" + ], + "requires": { + "linkedservices": { + "inputblob": { + "supportTypes": [ + "AzureBlobStorage" + ] + } + } + }, + "author": "bbedon@analytics.pe", + "annotations": [], + "services": [], + "categories": [] +} \ No newline at end of file