diff --git a/data_factory/dataset/diabetesCSV.json b/data_factory/dataset/diabetesCSV.json new file mode 100644 index 00000000..5e511221 --- /dev/null +++ b/data_factory/dataset/diabetesCSV.json @@ -0,0 +1,68 @@ +{ + "name": "diabetesCSV", + "properties": { + "linkedServiceName": { + "referenceName": "inputblob", + "type": "LinkedServiceReference" + }, + "annotations": [], + "type": "DelimitedText", + "typeProperties": { + "location": { + "type": "AzureBlobStorageLocation", + "fileName": "diabetes.csv", + "container": "datainput" + }, + "columnDelimiter": ",", + "escapeChar": "\\", + "firstRowAsHeader": true, + "quoteChar": "\"" + }, + "schema": [ + { + "name": "AGE", + "type": "String" + }, + { + "name": "SEX", + "type": "String" + }, + { + "name": "BMI", + "type": "String" + }, + { + "name": "BP", + "type": "String" + }, + { + "name": "S1", + "type": "String" + }, + { + "name": "S2", + "type": "String" + }, + { + "name": "S3", + "type": "String" + }, + { + "name": "S4", + "type": "String" + }, + { + "name": "S5", + "type": "String" + }, + { + "name": "S6", + "type": "String" + }, + { + "name": "Y", + "type": "String" + } + ] + } +} \ No newline at end of file diff --git a/data_factory/linkedService/inputblob.json b/data_factory/linkedService/inputblob.json new file mode 100644 index 00000000..7c92d260 --- /dev/null +++ b/data_factory/linkedService/inputblob.json @@ -0,0 +1,12 @@ +{ + "name": "inputblob", + "type": "Microsoft.DataFactory/factories/linkedservices", + "properties": { + "annotations": [], + "type": "AzureBlobStorage", + "typeProperties": { + "connectionString": "DefaultEndpointsProtocol=https;AccountName=brumlopsamlsa;EndpointSuffix=core.windows.net;", + "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkRGLU1MT1BTX2ViYWQwYzRhLTZlOTktNDk1OC1iMTE0LTkzOGVkNGY5ZDliMSINCn0=" + } + } +} \ No newline at end of file diff --git a/data_factory/pipeline/data to mlops.json b/data_factory/pipeline/data to mlops.json new file mode 100644 index 00000000..6916b655 --- /dev/null +++ b/data_factory/pipeline/data to mlops.json @@ -0,0 +1,57 @@ +{ + "name": "data to mlops", + "properties": { + "activities": [ + { + "name": "data to ml model", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "DelimitedTextSource", + "storeSettings": { + "type": "AzureBlobStorageReadSettings", + "recursive": true + }, + "formatSettings": { + "type": "DelimitedTextReadSettings" + } + }, + "sink": { + "type": "DelimitedTextSink", + "storeSettings": { + "type": "AzureBlobStorageWriteSettings" + }, + "formatSettings": { + "type": "DelimitedTextWriteSettings", + "quoteAllText": true, + "fileExtension": ".txt" + } + }, + "enableStaging": false + }, + "inputs": [ + { + "referenceName": "diabetesCSV", + "type": "DatasetReference" + } + ], + "outputs": [ + { + "referenceName": "diabetesCSV", + "type": "DatasetReference" + } + ] + } + ], + "annotations": [] + } +} \ No newline at end of file diff --git a/data_factory/temp.txt b/data_factory/temp.txt new file mode 100644 index 00000000..e69de29b diff --git a/data_factory/templates/data_to_mlops/data_to_mlops.json b/data_factory/templates/data_to_mlops/data_to_mlops.json new file mode 100644 index 00000000..5f9f81fa --- /dev/null +++ b/data_factory/templates/data_to_mlops/data_to_mlops.json @@ -0,0 +1,151 @@ +{ + "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "factoryName": { + "type": "string", + "metadata": "Data Factory name" + }, + "inputblob": { + "type": "string" + } + }, + "variables": { + "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" + }, + "resources": [ + { + "name": "[concat(parameters('factoryName'), '/data to mlops')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "data to ml model", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "DelimitedTextSource", + "storeSettings": { + "type": "AzureBlobStorageReadSettings", + "recursive": true + }, + "formatSettings": { + "type": "DelimitedTextReadSettings" + } + }, + "sink": { + "type": "DelimitedTextSink", + "storeSettings": { + "type": "AzureBlobStorageWriteSettings" + }, + "formatSettings": { + "type": "DelimitedTextWriteSettings", + "quoteAllText": true, + "fileExtension": ".txt" + } + }, + "enableStaging": false + }, + "inputs": [ + { + "referenceName": "diabetesCSV", + "type": "DatasetReference" + } + ], + "outputs": [ + { + "referenceName": "diabetesCSV", + "type": "DatasetReference" + } + ] + } + ], + "annotations": [] + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/datasets/diabetesCSV')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/diabetesCSV')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "[parameters('inputblob')]", + "type": "LinkedServiceReference" + }, + "annotations": [], + "type": "DelimitedText", + "typeProperties": { + "location": { + "type": "AzureBlobStorageLocation", + "fileName": "diabetes.csv", + "container": "datainput" + }, + "columnDelimiter": ",", + "escapeChar": "\\", + "firstRowAsHeader": true, + "quoteChar": "\"" + }, + "schema": [ + { + "name": "AGE", + "type": "String" + }, + { + "name": "SEX", + "type": "String" + }, + { + "name": "BMI", + "type": "String" + }, + { + "name": "BP", + "type": "String" + }, + { + "name": "S1", + "type": "String" + }, + { + "name": "S2", + "type": "String" + }, + { + "name": "S3", + "type": "String" + }, + { + "name": "S4", + "type": "String" + }, + { + "name": "S5", + "type": "String" + }, + { + "name": "S6", + "type": "String" + }, + { + "name": "Y", + "type": "String" + } + ] + }, + "dependsOn": [] + } + ] +} \ No newline at end of file diff --git a/data_factory/templates/data_to_mlops/manifest.json b/data_factory/templates/data_to_mlops/manifest.json new file mode 100644 index 00000000..b845b734 --- /dev/null +++ b/data_factory/templates/data_to_mlops/manifest.json @@ -0,0 +1,21 @@ +{ + "name": "data_to_mlops", + "image": "Copy datadata to ml model", + "icons": [ + "Copy", + "DelimitedText" + ], + "requires": { + "linkedservices": { + "inputblob": { + "supportTypes": [ + "AzureBlobStorage" + ] + } + } + }, + "author": "bbedon@analytics.pe", + "annotations": [], + "services": [], + "categories": [] +} \ No newline at end of file diff --git a/diabetes_regression/training/train.py b/diabetes_regression/training/train.py index 22258042..acd1886d 100644 --- a/diabetes_regression/training/train.py +++ b/diabetes_regression/training/train.py @@ -24,6 +24,8 @@ POSSIBILITY OF SUCH DAMAGE. """ +# Importación de Librerias + import os import pandas as pd from sklearn.linear_model import Ridge diff --git a/ml_service/pipelines/load_sample_data.py b/ml_service/pipelines/load_sample_data.py index cad56568..717fc7ab 100644 --- a/ml_service/pipelines/load_sample_data.py +++ b/ml_service/pipelines/load_sample_data.py @@ -5,7 +5,7 @@ # Loads the diabetes sample data from sklearn and produces a csv file that can # be used by the build/train pipeline script. -def create_sample_data_csv(file_name): +def create_sample_data_csv(): sample_data = load_diabetes() df = pd.DataFrame( data=sample_data.data,