From 1a383c530206d38bdb2c425dafb3837190eb2d13 Mon Sep 17 00:00:00 2001 From: Quentin WEPHRE Date: Fri, 19 Jul 2024 08:59:28 +0200 Subject: [PATCH] Upgrade to 1.5.2, tooling for VPN API requests --- .gitignore | 2 + Azure/bash_direct_method.sh | 42 +- Azure/bash_direct_method_2.sh | 44 +- Azure/bash_direct_method_3.sh | 46 +- Azure/create-moxa-list.py | 168 ++--- Azure/create-moxa.py | 96 +-- Azure/create_moxa.txt | 8 +- Azure/moxa_ac_template_1.2.json | 142 ++-- Azure/moxa_ac_template_1.3.json | 142 ++-- Azure/moxa_ac_template_1.4.json | 142 ++-- Azure/moxa_ac_template_1.5.json | 142 ++-- Azure/moxa_ac_template_1.5_patch.json | 142 ++-- Moulinette/LICENSE | 44 +- Moulinette/README.md | 190 +++--- Moulinette/data_path_config.py | 938 +++++++++++++------------- Moulinette/requirements.txt | 4 +- Python/azure_iot_hub_1.py | 92 +-- Python/danish_D1_api.py | 288 ++++---- Python/danish_batch_api.py | 449 ++++++++---- Python/danish_batch_scp.py | 216 +++--- Python/danish_batch_ssh.py | 174 ++--- Python/server_updating.py | 166 ++--- Python/stat_json_buffer.py | 154 ++--- Python/tranform_export_hierarchy.py | 28 + README.md | 2 +- 25 files changed, 2026 insertions(+), 1835 deletions(-) create mode 100644 .gitignore create mode 100644 Python/tranform_export_hierarchy.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6caaf75 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +I-Sight_Generated_Files* +DATAMODEL_* \ No newline at end of file diff --git a/Azure/bash_direct_method.sh b/Azure/bash_direct_method.sh index 1e5e6a8..e870f44 100644 --- a/Azure/bash_direct_method.sh +++ b/Azure/bash_direct_method.sh @@ -1,21 +1,21 @@ -#!/bin/bash - -# Get all devices from an IoT Hub (iothub_name) that have the given tag (tag_key == tag_value) -# Can also filter using tags.number -# Execute an Azure command for each of the devices found -# Here the command will execute a Direct Method for the ThingsPro module on the devices. This Direct Method enable the Discovery Service. - -iothub_name="IotHub-CUBE-Prod" - -tag_key="site" -tag_value="DANISH" - -devices=$(az iot hub query --hub-name $iothub_name --query-command "SELECT * FROM devices WHERE tags.$tag_key = '$tag_value' AND capabilities.iotEdge = true" --output json) - -device_ids=$(echo "$devices" | jq -r '.[].deviceId') - -for device_id in $device_ids -do - echo "$device_id" - az iot hub invoke-module-method --method-name thingspro-api-v1 --method-payload "{\"method\":\"PUT\",\"path\":\"/system/discovery\",\"requestBody\":{\"enable\": true}""}" --device-id $device_id --module-id thingspro-agent --hub-name $iothub_name -done +#!/bin/bash + +# Get all devices from an IoT Hub (iothub_name) that have the given tag (tag_key == tag_value) +# Can also filter using tags.number +# Execute an Azure command for each of the devices found +# Here the command will execute a Direct Method for the ThingsPro module on the devices. This Direct Method enable the Discovery Service. + +iothub_name="IotHub-CUBE-Prod" + +tag_key="site" +tag_value="DANISH" + +devices=$(az iot hub query --hub-name $iothub_name --query-command "SELECT * FROM devices WHERE tags.$tag_key = '$tag_value' AND capabilities.iotEdge = true" --output json) + +device_ids=$(echo "$devices" | jq -r '.[].deviceId') + +for device_id in $device_ids +do + echo "$device_id" + az iot hub invoke-module-method --method-name thingspro-api-v1 --method-payload "{\"method\":\"PUT\",\"path\":\"/system/discovery\",\"requestBody\":{\"enable\": true}""}" --device-id $device_id --module-id thingspro-agent --hub-name $iothub_name +done diff --git a/Azure/bash_direct_method_2.sh b/Azure/bash_direct_method_2.sh index 01ee203..9ecef9a 100644 --- a/Azure/bash_direct_method_2.sh +++ b/Azure/bash_direct_method_2.sh @@ -1,23 +1,23 @@ -#!/bin/bash - -# Get all devices from an IoT Hub (iothub_name) that have the given tag (tag_key == tag_value) -# Can also filter using tags.number -# Execute an Azure command for each of the devices found -# Here the command will configure the modules of all the IoT Edge devices of Danish (as Danish have up to 29 devices) with the given template. - -iothub_name="IotHub-CUBE-Prod" - -tag_key="site" -tag_value="DANISH" - -json_output=$(az iot hub query --hub-name $iothub_name --query-command "SELECT * FROM devices WHERE tags.$tag_key = '$tag_value' AND tags.number != '30' AND capabilities.iotEdge = true" --output json) - -hashmap=$(echo "$json_output" | jq -r 'map({key: .tags.number, value: .deviceId}) | from_entries') - -for key in $(echo "$hashmap" | jq -r 'keys | map(tonumber) | sort_by(.) | .[]'); do - value=$(jq -r --arg k "$key" '.[$k]' <<< "$hashmap") - - az iot edge set-modules --device-id $value --hub-name $iothub_name --content moxa_ac_template_1.5_patch.json -done - +#!/bin/bash + +# Get all devices from an IoT Hub (iothub_name) that have the given tag (tag_key == tag_value) +# Can also filter using tags.number +# Execute an Azure command for each of the devices found +# Here the command will configure the modules of all the IoT Edge devices of Danish (as Danish have up to 29 devices) with the given template. + +iothub_name="IotHub-CUBE-Prod" + +tag_key="site" +tag_value="DANISH" + +json_output=$(az iot hub query --hub-name $iothub_name --query-command "SELECT * FROM devices WHERE tags.$tag_key = '$tag_value' AND tags.number != '30' AND capabilities.iotEdge = true" --output json) + +hashmap=$(echo "$json_output" | jq -r 'map({key: .tags.number, value: .deviceId}) | from_entries') + +for key in $(echo "$hashmap" | jq -r 'keys | map(tonumber) | sort_by(.) | .[]'); do + value=$(jq -r --arg k "$key" '.[$k]' <<< "$hashmap") + + az iot edge set-modules --device-id $value --hub-name $iothub_name --content moxa_ac_template_1.5_patch.json +done + echo $hashmap \ No newline at end of file diff --git a/Azure/bash_direct_method_3.sh b/Azure/bash_direct_method_3.sh index 715dfca..c2d582b 100644 --- a/Azure/bash_direct_method_3.sh +++ b/Azure/bash_direct_method_3.sh @@ -1,23 +1,23 @@ -#!/bin/bash - -# Get all devices from an IoT Hub (iothub_name) that have the given tag (tag_key == tag_value) -# Can also filter using tags.number -# Execute an Azure command for each of the devices found -# Here the command will execute a Direct Method for the ThingsPro module on the devices. - -iothub_name="IotHub-CUBE-Prod" - -tag_key="site" -tag_value="DANISH" - -json_output=$(az iot hub query --hub-name $iothub_name --query-command "SELECT * FROM devices WHERE tags.$tag_key = '$tag_value' AND (tags.number = '6' OR tags.number = '8' OR tags.number = '12' OR tags.number = '13') AND capabilities.iotEdge = true" --output json) - -hashmap=$(echo "$json_output" | jq -r 'map({key: .tags.number, value: .deviceId}) | from_entries') - -for key in $(echo "$hashmap" | jq -r 'keys | map(tonumber) | sort_by(.) | .[]'); do - value=$(jq -r --arg k "$key" '.[$k]' <<< "$hashmap") - status=$(az iot hub invoke-module-method --timeout 10 --method-name thingspro-api-v1 --method-payload '{"method":"GET","path":"/device/general"}' --device-id $value --module-id thingspro-agent --hub-name $iothub_name | jq .status) - echo "Key: $key, Value: $value, Status: $status" -done - -#echo $hashmap +#!/bin/bash + +# Get all devices from an IoT Hub (iothub_name) that have the given tag (tag_key == tag_value) +# Can also filter using tags.number +# Execute an Azure command for each of the devices found +# Here the command will execute a Direct Method for the ThingsPro module on the devices. + +iothub_name="IotHub-CUBE-Prod" + +tag_key="site" +tag_value="DANISH" + +json_output=$(az iot hub query --hub-name $iothub_name --query-command "SELECT * FROM devices WHERE tags.$tag_key = '$tag_value' AND (tags.number = '6' OR tags.number = '8' OR tags.number = '12' OR tags.number = '13') AND capabilities.iotEdge = true" --output json) + +hashmap=$(echo "$json_output" | jq -r 'map({key: .tags.number, value: .deviceId}) | from_entries') + +for key in $(echo "$hashmap" | jq -r 'keys | map(tonumber) | sort_by(.) | .[]'); do + value=$(jq -r --arg k "$key" '.[$k]' <<< "$hashmap") + status=$(az iot hub invoke-module-method --timeout 10 --method-name thingspro-api-v1 --method-payload '{"method":"GET","path":"/device/general"}' --device-id $value --module-id thingspro-agent --hub-name $iothub_name | jq .status) + echo "Key: $key, Value: $value, Status: $status" +done + +#echo $hashmap diff --git a/Azure/create-moxa-list.py b/Azure/create-moxa-list.py index 5085731..c06c64e 100644 --- a/Azure/create-moxa-list.py +++ b/Azure/create-moxa-list.py @@ -1,84 +1,84 @@ -import pandas as pd -import subprocess -import argparse -import sys -import json - -# This Python script will read an Excel file containing a list of devices for a site (using --serials *file name*) -# It will then create a IoT Edge device for each of the line in the file -# The user need to precise which IoT Hub is used (using --env *PROD or DEV*) -# The user need provide the site name, which will be added as a tag (using --site *site name*) -# The user need to precise which configuration of the modules should be used, according to the running firmware on the devices (using --version *version number*) -# Example: -# python create_devices_list.py --serials DANISH.xlsx --env PROD --site DANISH --version 1.5_patch -# will create all the IoT Edge devices using the serial number found in the DANISH.xlsx file -# on the PROD IoT Hub -# tag each new device with a site = DANISH -# configure the modules for each devices with the corresponding version found in moxa_ac_template_1.5_patch.json -# each device will have also be given a number = i where i is incremented automatically according to the number of device in the Excel file -# the provided Excel file will be modified, adding a column connection_string for each of the devices created with their corresponding connection string -# the output is therefore SENSITIVE - -def generate_commands(serials_file, env, site, version): - # Read serial numbers from Excel file - df = pd.read_excel(serials_file) - df = df[df['device_name'].notnull()] - serials = df['device_name'].tolist() - - # Initialize number - number = 1 - - # List to store connection strings - connection_strings = [] - - # Generate commands for each serial number - for serial in serials: - # Replace placeholders with actual values - print(serial, end=" ") - device_id = f"DIGIT-{serial}" - print(device_id, end=" ") - tags = f'{{"deviceId":"{device_id}","site":"{site}","number":"{number}"}}' - content = f"moxa_ac_template_{version}.json" - # Construct command strings - create_command = f"az iot hub device-identity create --device-id {device_id} --hub-name IotHub-CUBE-{env} --edge-enabled" - twin_update_command = f"az iot hub device-twin update --device-id {device_id} --hub-name IotHub-CUBE-{env} --set tags='{tags}'" - set_modules_command = f"az iot edge set-modules --device-id {device_id} --hub-name IotHub-CUBE-{env} --content {content}" - - # Execute create command and get primary key - create_output = subprocess.check_output(create_command, shell=True) - create_output_json = json.loads(create_output.decode('utf-8')) - primary_key = create_output_json['authentication']['symmetricKey']['primaryKey'] - print(primary_key, end=" ") - - # Generate connection string - connection_string = f"HostName=IotHub-CUBE-{env}.azure-devices.net;DeviceId={device_id};SharedAccessKey={primary_key}" - print(connection_string) - connection_strings.append(connection_string) - - tags_output = subprocess.run(twin_update_command, shell=True) - modules_output = subprocess.run(set_modules_command, shell=True) - # Increment number - number += 1 - - # Add connection strings to DataFrame - df['connection_string'] = connection_strings - - # Save DataFrame to Excel file - df.to_excel(serials_file, index=False) - -if __name__ == "__main__": - # Parse command line arguments - parser = argparse.ArgumentParser(description='Create devices list') - parser.add_argument('--serials', required=True, help='Excel file containing serial numbers') - parser.add_argument('--env', required=True, help='Environment (PROD or DEV)') - parser.add_argument('--site', required=True, help='Site name') - parser.add_argument('--version', required=True, help='Version number') - args = parser.parse_args() - - if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) - - # Generate commands - generate_commands(args.serials, args.env, args.site, args.version) - +import pandas as pd +import subprocess +import argparse +import sys +import json + +# This Python script will read an Excel file containing a list of devices for a site (using --serials *file name*) +# It will then create a IoT Edge device for each of the line in the file +# The user need to precise which IoT Hub is used (using --env *PROD or DEV*) +# The user need provide the site name, which will be added as a tag (using --site *site name*) +# The user need to precise which configuration of the modules should be used, according to the running firmware on the devices (using --version *version number*) +# Example: +# python create_devices_list.py --serials DANISH.xlsx --env PROD --site DANISH --version 1.5_patch +# will create all the IoT Edge devices using the serial number found in the DANISH.xlsx file +# on the PROD IoT Hub +# tag each new device with a site = DANISH +# configure the modules for each devices with the corresponding version found in moxa_ac_template_1.5_patch.json +# each device will have also be given a number = i where i is incremented automatically according to the number of device in the Excel file +# the provided Excel file will be modified, adding a column connection_string for each of the devices created with their corresponding connection string +# the output is therefore SENSITIVE + +def generate_commands(serials_file, env, site, version): + # Read serial numbers from Excel file + df = pd.read_excel(serials_file) + df = df[df['device_name'].notnull()] + serials = df['device_name'].tolist() + + # Initialize number + number = 1 + + # List to store connection strings + connection_strings = [] + + # Generate commands for each serial number + for serial in serials: + # Replace placeholders with actual values + print(serial, end=" ") + device_id = f"DIGIT-{serial}" + print(device_id, end=" ") + tags = f'{{"deviceId":"{device_id}","site":"{site}","number":"{number}"}}' + content = f"moxa_ac_template_{version}.json" + # Construct command strings + create_command = f"az iot hub device-identity create --device-id {device_id} --hub-name IotHub-CUBE-{env} --edge-enabled" + twin_update_command = f"az iot hub device-twin update --device-id {device_id} --hub-name IotHub-CUBE-{env} --set tags='{tags}'" + set_modules_command = f"az iot edge set-modules --device-id {device_id} --hub-name IotHub-CUBE-{env} --content {content}" + + # Execute create command and get primary key + create_output = subprocess.check_output(create_command, shell=True) + create_output_json = json.loads(create_output.decode('utf-8')) + primary_key = create_output_json['authentication']['symmetricKey']['primaryKey'] + print(primary_key, end=" ") + + # Generate connection string + connection_string = f"HostName=IotHub-CUBE-{env}.azure-devices.net;DeviceId={device_id};SharedAccessKey={primary_key}" + print(connection_string) + connection_strings.append(connection_string) + + tags_output = subprocess.run(twin_update_command, shell=True) + modules_output = subprocess.run(set_modules_command, shell=True) + # Increment number + number += 1 + + # Add connection strings to DataFrame + df['connection_string'] = connection_strings + + # Save DataFrame to Excel file + df.to_excel(serials_file, index=False) + +if __name__ == "__main__": + # Parse command line arguments + parser = argparse.ArgumentParser(description='Create devices list') + parser.add_argument('--serials', required=True, help='Excel file containing serial numbers') + parser.add_argument('--env', required=True, help='Environment (PROD or DEV)') + parser.add_argument('--site', required=True, help='Site name') + parser.add_argument('--version', required=True, help='Version number') + args = parser.parse_args() + + if len(sys.argv) == 1: + parser.print_help() + sys.exit(1) + + # Generate commands + generate_commands(args.serials, args.env, args.site, args.version) + diff --git a/Azure/create-moxa.py b/Azure/create-moxa.py index 050b205..2da93b7 100644 --- a/Azure/create-moxa.py +++ b/Azure/create-moxa.py @@ -1,48 +1,48 @@ -import pandas as pd -import subprocess -import argparse -import sys -import json - -# This Python script will create an IoT Edge device according to the provided parameters -# It will create the device with the name DIGIT-*serial number* (using --serial *serial number*) -# The user need to precise the number of the gateway on the site (using --number *number*) -# The user need to precise which IoT Hub is used (using --env *PROD or DEV*) -# The user need provide the site name, which will be added as a tag (using --site *site name*) -# The user need to precise which configuration of the modules should be used, according to the running firmware on the device (using --version *version number*) -# Example: -# python create_moxa.py --serial TBBHB1044382 --number 5 --env PROD --site DANISH --version 1.5_patch -# will create the IoT Edge device using the serial number provided (TBBHB1044382) -# on the PROD IoT Hub -# tag the device with number = 5 -# tag the device with site = DANISH -# configure the modules for the device with the corresponding version found in moxa_ac_template_1.5_patch.json - -def generate_commands(serial, number, env, site, version): - - device_id = f"DIGIT-{serial}" - tags = f'{{"deviceId":"{device_id}","site":"{site}","number":"{number}"}}' - content = f"moxa_ac_template_{version}.json" - create_device_command = f"az iot hub device-identity create --device-id {device_id} --hub-name IotHub-CUBE-{env} --edge-enabled" - twin_update_command = f"az iot hub device-twin update --device-id {device_id} --hub-name IotHub-CUBE-{env} --set tags='{tags}'" - set_modules_command = f"az iot edge set-modules --device-id {device_id} --hub-name IotHub-CUBE-{env} --content {content}" - subprocess.run(create_device_command, shell=True) - subprocess.run(twin_update_command, shell=True) - subprocess.run(set_modules_command, shell=True) - -if __name__ == "__main__": - # Parse command line arguments - parser = argparse.ArgumentParser(description='Create devices list') - parser.add_argument('--serial', required=True, help='Serial number of the gateway') - parser.add_argument('--number', required=True, help='Gateway on-site number') - parser.add_argument('--env', required=True, help='Environment (PROD or DEV)') - parser.add_argument('--site', required=True, help='Site name') - parser.add_argument('--version', required=True, help='Version number') - args = parser.parse_args() - - if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) - - # Generate commands - generate_commands(args.serial, args.number, args.env, args.site, args.version) +import pandas as pd +import subprocess +import argparse +import sys +import json + +# This Python script will create an IoT Edge device according to the provided parameters +# It will create the device with the name DIGIT-*serial number* (using --serial *serial number*) +# The user need to precise the number of the gateway on the site (using --number *number*) +# The user need to precise which IoT Hub is used (using --env *PROD or DEV*) +# The user need provide the site name, which will be added as a tag (using --site *site name*) +# The user need to precise which configuration of the modules should be used, according to the running firmware on the device (using --version *version number*) +# Example: +# python create_moxa.py --serial TBBHB1044382 --number 5 --env PROD --site DANISH --version 1.5_patch +# will create the IoT Edge device using the serial number provided (TBBHB1044382) +# on the PROD IoT Hub +# tag the device with number = 5 +# tag the device with site = DANISH +# configure the modules for the device with the corresponding version found in moxa_ac_template_1.5_patch.json + +def generate_commands(serial, number, env, site, version): + + device_id = f"DIGIT-{serial}" + tags = f'{{"deviceId":"{device_id}","site":"{site}","number":"{number}"}}' + content = f"moxa_ac_template_{version}.json" + create_device_command = f"az iot hub device-identity create --device-id {device_id} --hub-name IotHub-CUBE-{env} --edge-enabled" + twin_update_command = f"az iot hub device-twin update --device-id {device_id} --hub-name IotHub-CUBE-{env} --set tags='{tags}'" + set_modules_command = f"az iot edge set-modules --device-id {device_id} --hub-name IotHub-CUBE-{env} --content {content}" + subprocess.run(create_device_command, shell=True) + subprocess.run(twin_update_command, shell=True) + subprocess.run(set_modules_command, shell=True) + +if __name__ == "__main__": + # Parse command line arguments + parser = argparse.ArgumentParser(description='Create devices list') + parser.add_argument('--serial', required=True, help='Serial number of the gateway') + parser.add_argument('--number', required=True, help='Gateway on-site number') + parser.add_argument('--env', required=True, help='Environment (PROD or DEV)') + parser.add_argument('--site', required=True, help='Site name') + parser.add_argument('--version', required=True, help='Version number') + args = parser.parse_args() + + if len(sys.argv) == 1: + parser.print_help() + sys.exit(1) + + # Generate commands + generate_commands(args.serial, args.number, args.env, args.site, args.version) diff --git a/Azure/create_moxa.txt b/Azure/create_moxa.txt index c055d49..2165c9f 100644 --- a/Azure/create_moxa.txt +++ b/Azure/create_moxa.txt @@ -1,5 +1,5 @@ -az iot hub device-identity create --device-id DIGIT-TBBHB1044382 --hub-name IotHub-CUBE-PROD --edge-enabled -az iot hub device-twin update --device-id DIGIT-TBBHB1044382 --hub-name IotHub-CUBE-PROD --set tags='{"deviceId":"DIGIT-TBBHB1044382","site":"SASK","number":"1"}' -az iot edge set-modules --device-id DIGIT-TBBHB1044382 --hub-name IotHub-CUBE-DEV --content moxa_ac_template.json - +az iot hub device-identity create --device-id DIGIT-TBBHB1044382 --hub-name IotHub-CUBE-PROD --edge-enabled +az iot hub device-twin update --device-id DIGIT-TBBHB1044382 --hub-name IotHub-CUBE-PROD --set tags='{"deviceId":"DIGIT-TBBHB1044382","site":"SASK","number":"1"}' +az iot edge set-modules --device-id DIGIT-TBBHB1044382 --hub-name IotHub-CUBE-DEV --content moxa_ac_template.json + This file is a just a note for usual Azure commands involved to create a device. \ No newline at end of file diff --git a/Azure/moxa_ac_template_1.2.json b/Azure/moxa_ac_template_1.2.json index 4d22529..5a86282 100644 --- a/Azure/moxa_ac_template_1.2.json +++ b/Azure/moxa_ac_template_1.2.json @@ -1,71 +1,71 @@ -{ - "modulesContent": { - "$edgeAgent": { - "properties.desired": { - "schemaVersion": "1.1", - "runtime": { - "type": "docker", - "settings": {} - }, - "systemModules": { - "edgeAgent": { - "env": { - "UpstreamProtocol": { - "value": "AMQPWS" - }, - "SendRuntimeQualityTelemetry": { - "value": false - } - }, - "settings": { - "image": "mcr.microsoft.com/azureiotedge-agent:1.0.10", - "createOptions": "{\"HostConfig\":{\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"3\"}}}}" - }, - "type": "docker" - }, - "edgeHub": { - "env": { - "UpstreamProtocol": { - "value": "AMQPWS" - } - }, - "restartPolicy": "always", - "settings": { - "image": "mcr.microsoft.com/azureiotedge-hub:1.0.10", - "createOptions": "{\"HostConfig\":{\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"3\"}},\"PortBindings\":{\"443/tcp\":[{\"HostPort\":\"443\"}],\"5671/tcp\":[{\"HostPort\":\"5671\"}],\"8883/tcp\":[{\"HostPort\":\"8883\"}]}}}" - }, - "status": "running", - "type": "docker" - } - }, - "modules": { - "thingspro-agent": { - "restartPolicy": "always", - "settings": { - "image": "moxa2019/thingspro-agent:2.1.1-armhf", - "createOptions": "{\"HostConfig\":{\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"3\"}},\"Binds\":[\"/var/thingspro/apps/cloud/data/setting/:/var/thingspro/cloud/setting/\",\"/run/:/host/run/\",\"/var/thingspro/data/:/var/thingspro/data/\"]}}" - }, - "status": "running", - "type": "docker" - } - } - } - }, - "$edgeHub": { - "properties.desired": { - "schemaVersion": "1.1", - "storeAndForwardConfiguration": { - "timeToLiveSecs": 86400 - }, - "routes": { - "route": { - "route": "FROM /messages/* INTO $upstream" - } - } - } - }, - "thingspro-agent": { - "properties.desired": {} - } - } -} +{ + "modulesContent": { + "$edgeAgent": { + "properties.desired": { + "schemaVersion": "1.1", + "runtime": { + "type": "docker", + "settings": {} + }, + "systemModules": { + "edgeAgent": { + "env": { + "UpstreamProtocol": { + "value": "AMQPWS" + }, + "SendRuntimeQualityTelemetry": { + "value": false + } + }, + "settings": { + "image": "mcr.microsoft.com/azureiotedge-agent:1.0.10", + "createOptions": "{\"HostConfig\":{\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"3\"}}}}" + }, + "type": "docker" + }, + "edgeHub": { + "env": { + "UpstreamProtocol": { + "value": "AMQPWS" + } + }, + "restartPolicy": "always", + "settings": { + "image": "mcr.microsoft.com/azureiotedge-hub:1.0.10", + "createOptions": "{\"HostConfig\":{\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"3\"}},\"PortBindings\":{\"443/tcp\":[{\"HostPort\":\"443\"}],\"5671/tcp\":[{\"HostPort\":\"5671\"}],\"8883/tcp\":[{\"HostPort\":\"8883\"}]}}}" + }, + "status": "running", + "type": "docker" + } + }, + "modules": { + "thingspro-agent": { + "restartPolicy": "always", + "settings": { + "image": "moxa2019/thingspro-agent:2.1.1-armhf", + "createOptions": "{\"HostConfig\":{\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"3\"}},\"Binds\":[\"/var/thingspro/apps/cloud/data/setting/:/var/thingspro/cloud/setting/\",\"/run/:/host/run/\",\"/var/thingspro/data/:/var/thingspro/data/\"]}}" + }, + "status": "running", + "type": "docker" + } + } + } + }, + "$edgeHub": { + "properties.desired": { + "schemaVersion": "1.1", + "storeAndForwardConfiguration": { + "timeToLiveSecs": 86400 + }, + "routes": { + "route": { + "route": "FROM /messages/* INTO $upstream" + } + } + } + }, + "thingspro-agent": { + "properties.desired": {} + } + } +} diff --git a/Azure/moxa_ac_template_1.3.json b/Azure/moxa_ac_template_1.3.json index ddabf33..cb1c7db 100644 --- a/Azure/moxa_ac_template_1.3.json +++ b/Azure/moxa_ac_template_1.3.json @@ -1,71 +1,71 @@ -{ - "modulesContent": { - "$edgeAgent": { - "properties.desired": { - "schemaVersion": "1.1", - "runtime": { - "type": "docker", - "settings": {} - }, - "systemModules": { - "edgeAgent": { - "env": { - "UpstreamProtocol": { - "value": "AMQPWS" - }, - "SendRuntimeQualityTelemetry": { - "value": false - } - }, - "settings": { - "image": "mcr.microsoft.com/azureiotedge-agent:1.1.4", - "createOptions": "{\"HostConfig\":{\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"3\"}}}}" - }, - "type": "docker" - }, - "edgeHub": { - "env": { - "UpstreamProtocol": { - "value": "AMQPWS" - } - }, - "restartPolicy": "always", - "settings": { - "image": "mcr.microsoft.com/azureiotedge-hub:1.1.4", - "createOptions": "{\"HostConfig\":{\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"3\"}},\"PortBindings\":{\"443/tcp\":[{\"HostPort\":\"443\"}],\"5671/tcp\":[{\"HostPort\":\"5671\"}],\"8883/tcp\":[{\"HostPort\":\"8883\"}]}}}" - }, - "status": "running", - "type": "docker" - } - }, - "modules": { - "thingspro-agent": { - "restartPolicy": "always", - "settings": { - "image": "moxa2019/thingspro-agent:2.2.3-armhf", - "createOptions": "{\"HostConfig\":{\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"3\"}},\"Binds\":[\"/var/thingspro/apps/cloud/data/setting/:/var/thingspro/cloud/setting/\",\"/run/:/host/run/\",\"/var/thingspro/data/:/var/thingspro/data/\"]}}" - }, - "status": "running", - "type": "docker" - } - } - } - }, - "$edgeHub": { - "properties.desired": { - "schemaVersion": "1.1", - "storeAndForwardConfiguration": { - "timeToLiveSecs": 86400 - }, - "routes": { - "route": { - "route": "FROM /messages/* INTO $upstream" - } - } - } - }, - "thingspro-agent": { - "properties.desired": {} - } - } -} +{ + "modulesContent": { + "$edgeAgent": { + "properties.desired": { + "schemaVersion": "1.1", + "runtime": { + "type": "docker", + "settings": {} + }, + "systemModules": { + "edgeAgent": { + "env": { + "UpstreamProtocol": { + "value": "AMQPWS" + }, + "SendRuntimeQualityTelemetry": { + "value": false + } + }, + "settings": { + "image": "mcr.microsoft.com/azureiotedge-agent:1.1.4", + "createOptions": "{\"HostConfig\":{\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"3\"}}}}" + }, + "type": "docker" + }, + "edgeHub": { + "env": { + "UpstreamProtocol": { + "value": "AMQPWS" + } + }, + "restartPolicy": "always", + "settings": { + "image": "mcr.microsoft.com/azureiotedge-hub:1.1.4", + "createOptions": "{\"HostConfig\":{\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"3\"}},\"PortBindings\":{\"443/tcp\":[{\"HostPort\":\"443\"}],\"5671/tcp\":[{\"HostPort\":\"5671\"}],\"8883/tcp\":[{\"HostPort\":\"8883\"}]}}}" + }, + "status": "running", + "type": "docker" + } + }, + "modules": { + "thingspro-agent": { + "restartPolicy": "always", + "settings": { + "image": "moxa2019/thingspro-agent:2.2.3-armhf", + "createOptions": "{\"HostConfig\":{\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"3\"}},\"Binds\":[\"/var/thingspro/apps/cloud/data/setting/:/var/thingspro/cloud/setting/\",\"/run/:/host/run/\",\"/var/thingspro/data/:/var/thingspro/data/\"]}}" + }, + "status": "running", + "type": "docker" + } + } + } + }, + "$edgeHub": { + "properties.desired": { + "schemaVersion": "1.1", + "storeAndForwardConfiguration": { + "timeToLiveSecs": 86400 + }, + "routes": { + "route": { + "route": "FROM /messages/* INTO $upstream" + } + } + } + }, + "thingspro-agent": { + "properties.desired": {} + } + } +} diff --git a/Azure/moxa_ac_template_1.4.json b/Azure/moxa_ac_template_1.4.json index 05f8357..bb8552a 100644 --- a/Azure/moxa_ac_template_1.4.json +++ b/Azure/moxa_ac_template_1.4.json @@ -1,71 +1,71 @@ -{ - "modulesContent": { - "$edgeAgent": { - "properties.desired": { - "schemaVersion": "1.1", - "runtime": { - "type": "docker", - "settings": {} - }, - "systemModules": { - "edgeAgent": { - "env": { - "UpstreamProtocol": { - "value": "AMQPWS" - }, - "SendRuntimeQualityTelemetry": { - "value": false - } - }, - "settings": { - "image": "mcr.microsoft.com/azureiotedge-agent:1.2.7", - "createOptions": "{\"HostConfig\":{\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"3\"}}}}" - }, - "type": "docker" - }, - "edgeHub": { - "env": { - "UpstreamProtocol": { - "value": "AMQPWS" - } - }, - "restartPolicy": "always", - "settings": { - "image": "mcr.microsoft.com/azureiotedge-hub:1.2.7", - "createOptions": "{\"HostConfig\":{\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"3\"}},\"PortBindings\":{\"443/tcp\":[{\"HostPort\":\"443\"}],\"5671/tcp\":[{\"HostPort\":\"5671\"}],\"8883/tcp\":[{\"HostPort\":\"8883\"}]}}}" - }, - "status": "running", - "type": "docker" - } - }, - "modules": { - "thingspro-agent": { - "restartPolicy": "always", - "settings": { - "image": "moxa2019/thingspro-agent:2.2.3-armhf", - "createOptions": "{\"HostConfig\":{\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"3\"}},\"Binds\":[\"/var/thingspro/apps/cloud/data/setting/:/var/thingspro/cloud/setting/\",\"/run/:/host/run/\",\"/var/thingspro/data/:/var/thingspro/data/\"]}}" - }, - "status": "running", - "type": "docker" - } - } - } - }, - "$edgeHub": { - "properties.desired": { - "schemaVersion": "1.1", - "storeAndForwardConfiguration": { - "timeToLiveSecs": 86400 - }, - "routes": { - "route": { - "route": "FROM /messages/* INTO $upstream" - } - } - } - }, - "thingspro-agent": { - "properties.desired": {} - } - } -} +{ + "modulesContent": { + "$edgeAgent": { + "properties.desired": { + "schemaVersion": "1.1", + "runtime": { + "type": "docker", + "settings": {} + }, + "systemModules": { + "edgeAgent": { + "env": { + "UpstreamProtocol": { + "value": "AMQPWS" + }, + "SendRuntimeQualityTelemetry": { + "value": false + } + }, + "settings": { + "image": "mcr.microsoft.com/azureiotedge-agent:1.2.7", + "createOptions": "{\"HostConfig\":{\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"3\"}}}}" + }, + "type": "docker" + }, + "edgeHub": { + "env": { + "UpstreamProtocol": { + "value": "AMQPWS" + } + }, + "restartPolicy": "always", + "settings": { + "image": "mcr.microsoft.com/azureiotedge-hub:1.2.7", + "createOptions": "{\"HostConfig\":{\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"3\"}},\"PortBindings\":{\"443/tcp\":[{\"HostPort\":\"443\"}],\"5671/tcp\":[{\"HostPort\":\"5671\"}],\"8883/tcp\":[{\"HostPort\":\"8883\"}]}}}" + }, + "status": "running", + "type": "docker" + } + }, + "modules": { + "thingspro-agent": { + "restartPolicy": "always", + "settings": { + "image": "moxa2019/thingspro-agent:2.2.3-armhf", + "createOptions": "{\"HostConfig\":{\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"3\"}},\"Binds\":[\"/var/thingspro/apps/cloud/data/setting/:/var/thingspro/cloud/setting/\",\"/run/:/host/run/\",\"/var/thingspro/data/:/var/thingspro/data/\"]}}" + }, + "status": "running", + "type": "docker" + } + } + } + }, + "$edgeHub": { + "properties.desired": { + "schemaVersion": "1.1", + "storeAndForwardConfiguration": { + "timeToLiveSecs": 86400 + }, + "routes": { + "route": { + "route": "FROM /messages/* INTO $upstream" + } + } + } + }, + "thingspro-agent": { + "properties.desired": {} + } + } +} diff --git a/Azure/moxa_ac_template_1.5.json b/Azure/moxa_ac_template_1.5.json index 29f7917..5a14872 100644 --- a/Azure/moxa_ac_template_1.5.json +++ b/Azure/moxa_ac_template_1.5.json @@ -1,71 +1,71 @@ -{ - "modulesContent": { - "$edgeAgent": { - "properties.desired": { - "schemaVersion": "1.1", - "runtime": { - "type": "docker", - "settings": {} - }, - "systemModules": { - "edgeAgent": { - "env": { - "UpstreamProtocol": { - "value": "AMQPWS" - }, - "SendRuntimeQualityTelemetry": { - "value": false - } - }, - "settings": { - "image": "mcr.microsoft.com/azureiotedge-agent:1.4.10", - "createOptions": "{\"HostConfig\":{\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"3\"}}}}" - }, - "type": "docker" - }, - "edgeHub": { - "env": { - "UpstreamProtocol": { - "value": "AMQPWS" - } - }, - "restartPolicy": "always", - "settings": { - "image": "mcr.microsoft.com/azureiotedge-hub:1.4.10", - "createOptions": "{\"HostConfig\":{\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"3\"}},\"PortBindings\":{\"443/tcp\":[{\"HostPort\":\"443\"}],\"5671/tcp\":[{\"HostPort\":\"5671\"}],\"8883/tcp\":[{\"HostPort\":\"8883\"}]}}}" - }, - "status": "running", - "type": "docker" - } - }, - "modules": { - "thingspro-agent": { - "restartPolicy": "always", - "settings": { - "image": "moxa2019/thingspro-agent:2.2.3-armhf", - "createOptions": "{\"HostConfig\":{\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"3\"}},\"Binds\":[\"/var/thingspro/apps/cloud/data/setting/:/var/thingspro/cloud/setting/\",\"/run/:/host/run/\",\"/var/thingspro/data/:/var/thingspro/data/\"]}}" - }, - "status": "running", - "type": "docker" - } - } - } - }, - "$edgeHub": { - "properties.desired": { - "schemaVersion": "1.1", - "storeAndForwardConfiguration": { - "timeToLiveSecs": 86400 - }, - "routes": { - "route": { - "route": "FROM /messages/* INTO $upstream" - } - } - } - }, - "thingspro-agent": { - "properties.desired": {} - } - } -} +{ + "modulesContent": { + "$edgeAgent": { + "properties.desired": { + "schemaVersion": "1.1", + "runtime": { + "type": "docker", + "settings": {} + }, + "systemModules": { + "edgeAgent": { + "env": { + "UpstreamProtocol": { + "value": "AMQPWS" + }, + "SendRuntimeQualityTelemetry": { + "value": false + } + }, + "settings": { + "image": "mcr.microsoft.com/azureiotedge-agent:1.4.10", + "createOptions": "{\"HostConfig\":{\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"3\"}}}}" + }, + "type": "docker" + }, + "edgeHub": { + "env": { + "UpstreamProtocol": { + "value": "AMQPWS" + } + }, + "restartPolicy": "always", + "settings": { + "image": "mcr.microsoft.com/azureiotedge-hub:1.4.10", + "createOptions": "{\"HostConfig\":{\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"3\"}},\"PortBindings\":{\"443/tcp\":[{\"HostPort\":\"443\"}],\"5671/tcp\":[{\"HostPort\":\"5671\"}],\"8883/tcp\":[{\"HostPort\":\"8883\"}]}}}" + }, + "status": "running", + "type": "docker" + } + }, + "modules": { + "thingspro-agent": { + "restartPolicy": "always", + "settings": { + "image": "moxa2019/thingspro-agent:2.2.3-armhf", + "createOptions": "{\"HostConfig\":{\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"3\"}},\"Binds\":[\"/var/thingspro/apps/cloud/data/setting/:/var/thingspro/cloud/setting/\",\"/run/:/host/run/\",\"/var/thingspro/data/:/var/thingspro/data/\"]}}" + }, + "status": "running", + "type": "docker" + } + } + } + }, + "$edgeHub": { + "properties.desired": { + "schemaVersion": "1.1", + "storeAndForwardConfiguration": { + "timeToLiveSecs": 86400 + }, + "routes": { + "route": { + "route": "FROM /messages/* INTO $upstream" + } + } + } + }, + "thingspro-agent": { + "properties.desired": {} + } + } +} diff --git a/Azure/moxa_ac_template_1.5_patch.json b/Azure/moxa_ac_template_1.5_patch.json index 32f3a4d..2cc9777 100644 --- a/Azure/moxa_ac_template_1.5_patch.json +++ b/Azure/moxa_ac_template_1.5_patch.json @@ -1,71 +1,71 @@ -{ - "modulesContent": { - "$edgeAgent": { - "properties.desired": { - "schemaVersion": "1.1", - "runtime": { - "type": "docker", - "settings": {} - }, - "systemModules": { - "edgeAgent": { - "env": { - "UpstreamProtocol": { - "value": "AMQPWS" - }, - "SendRuntimeQualityTelemetry": { - "value": false - } - }, - "settings": { - "image": "mcr.microsoft.com/azureiotedge-agent:1.4.27", - "createOptions": "{\"HostConfig\":{\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"3\"}}}}" - }, - "type": "docker" - }, - "edgeHub": { - "env": { - "UpstreamProtocol": { - "value": "AMQPWS" - } - }, - "restartPolicy": "always", - "settings": { - "image": "mcr.microsoft.com/azureiotedge-hub:1.4.27", - "createOptions": "{\"HostConfig\":{\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"3\"}},\"PortBindings\":{\"443/tcp\":[{\"HostPort\":\"443\"}],\"5671/tcp\":[{\"HostPort\":\"5671\"}],\"8883/tcp\":[{\"HostPort\":\"8883\"}]}}}" - }, - "status": "running", - "type": "docker" - } - }, - "modules": { - "thingspro-agent": { - "restartPolicy": "always", - "settings": { - "image": "moxa2019/thingspro-agent:2.2.5-armhf", - "createOptions": "{\"HostConfig\":{\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"3\"}},\"Binds\":[\"/var/thingspro/apps/azureiotedge/data/setting/:/var/thingspro/cloud/setting/\",\"/run/:/host/run/\",\"/var/thingspro/data/:/var/thingspro/data/\"]}}" - }, - "status": "running", - "type": "docker" - } - } - } - }, - "$edgeHub": { - "properties.desired": { - "schemaVersion": "1.1", - "storeAndForwardConfiguration": { - "timeToLiveSecs": 86400 - }, - "routes": { - "route": { - "route": "FROM /messages/* INTO $upstream" - } - } - } - }, - "thingspro-agent": { - "properties.desired": {} - } - } -} +{ + "modulesContent": { + "$edgeAgent": { + "properties.desired": { + "schemaVersion": "1.1", + "runtime": { + "type": "docker", + "settings": {} + }, + "systemModules": { + "edgeAgent": { + "env": { + "UpstreamProtocol": { + "value": "AMQPWS" + }, + "SendRuntimeQualityTelemetry": { + "value": false + } + }, + "settings": { + "image": "mcr.microsoft.com/azureiotedge-agent:1.4.27", + "createOptions": "{\"HostConfig\":{\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"3\"}}}}" + }, + "type": "docker" + }, + "edgeHub": { + "env": { + "UpstreamProtocol": { + "value": "AMQPWS" + } + }, + "restartPolicy": "always", + "settings": { + "image": "mcr.microsoft.com/azureiotedge-hub:1.4.27", + "createOptions": "{\"HostConfig\":{\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"3\"}},\"PortBindings\":{\"443/tcp\":[{\"HostPort\":\"443\"}],\"5671/tcp\":[{\"HostPort\":\"5671\"}],\"8883/tcp\":[{\"HostPort\":\"8883\"}]}}}" + }, + "status": "running", + "type": "docker" + } + }, + "modules": { + "thingspro-agent": { + "restartPolicy": "always", + "settings": { + "image": "moxa2019/thingspro-agent:2.2.5-armhf", + "createOptions": "{\"HostConfig\":{\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"3\"}},\"Binds\":[\"/var/thingspro/apps/azureiotedge/data/setting/:/var/thingspro/cloud/setting/\",\"/run/:/host/run/\",\"/var/thingspro/data/:/var/thingspro/data/\"]}}" + }, + "status": "running", + "type": "docker" + } + } + } + }, + "$edgeHub": { + "properties.desired": { + "schemaVersion": "1.1", + "storeAndForwardConfiguration": { + "timeToLiveSecs": 86400 + }, + "routes": { + "route": { + "route": "FROM /messages/* INTO $upstream" + } + } + } + }, + "thingspro-agent": { + "properties.desired": {} + } + } +} diff --git a/Moulinette/LICENSE b/Moulinette/LICENSE index fa4754a..01bfd62 100644 --- a/Moulinette/LICENSE +++ b/Moulinette/LICENSE @@ -1,23 +1,23 @@ -MIT License - -Copyright (c) 2023 Amjad Badar - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - +MIT License + +Copyright (c) 2023 Amjad Badar + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + Original code modified 2024 Quentin WEPHRE \ No newline at end of file diff --git a/Moulinette/README.md b/Moulinette/README.md index 7116a5b..a5da3f0 100644 --- a/Moulinette/README.md +++ b/Moulinette/README.md @@ -1,95 +1,95 @@ -# ORIGINAL README FROM MOXA -# COMPLETELY OUTDATED AS OF 06/2024 -# DOES NOT WORK WITH CURRENT VERSION -# KEPT FOR INFORMATION AND HISTORICAL DATA - -## Moxa AIG-301- Data Path Configuration -## Pre-requisites - - -The following two Excel files from Saft are required - -- [ ] 0000123704_SAFT_Generic_Data_Model.xlsx -- [ ] I-Sight_Project_Communication_Network_Config.xlsx - -Python Script to generate shell sript from Execel files - data_path_config.py - -Note: The python script and the excel files must be kept in the same directory. The files generated by the python script will be present inside the "Generated_Files" folder. -Also, make sure the data provided in the Excel sheet is correct. Else it can affect the configuration process. - -## Step1 - -- [ ] The Python Version used for the project development is: -''' - 3.10.9 -''' - -- [ ] Install all dependencies of the python by running following command: -``` - sudo pip3 install -r requirements.txt -``` - -- [ ] Execute python script in the directory where Execl files are located -``` - python3 data_path_config.py -``` - -OUTPUT: - -On successfull execution of the script, it generates the folder "Generated_Files". The folder contains: - -1. data_path_configuration_shell_script.sh -2. data_config_debug.log - -The "data_path_configuration_shell_script.sh" is the script that needs to be executed in the Moxa device for configuring the ThingsPro Edge software. -The "data_config_debug.log" is the log file generated after running the python script and it performs some basic error checks on the given excel files. - -## Step2 -- [ ] Copy shell script into Moxa gateway in home directory via File Transfer tool example [winscp](https://winscp.net/download/WinSCP-5.21.7-Setup.exe) files - -## Step3 -- [ ] Change file mode of shell script to make it executable by executing the following commands: -``` - sudo sed -i -e 's/\r$//' data_path_configuration_shell_script.sh - sudo chmod +x data_path_configuration_shell_script.sh - -``` -## Step4 -- [ ] Execute shell script in root directory, otheriwse tpfunc will not deployed -``` - sudo su - -``` -``` - ./data_path_configuration_shell_script.sh - -``` -## Step5 - -- [ ] A log report "data_shell_script.log" is generated in the same directory after executing the shell script "data_path_configuration_shell_script.sh". This log report provides information on whether the commands of the shell script were successful or not based on HTTP status codes. Some most common HTTP requests from the log are: - -``` - HTTP request 200 or 201 means that command was successfully executed - HTTP request 400 and above means that the command was NOT executed successfully - -``` -Note: An HTTP request of 400 and above can either mean that the data (which is obtained from the provided excel sheets) within the command is Incorrect or that the ThingsPro Edge software is already configured with that data. - -- [ ] Verify results on ThingsPro Edge webGUI - - 1) Check Modbus Configuration - 2) Check Azure IoT Edge Telemetry - - A seperate topic will be created for each slave - 3) Tpfunc under Function - -## Additional Information - - The "Generated_Files" folder in this repository was created by running the python script with the excel sheets in the same repository. It can be used for reference. - - The Folder "Screenshots" has images that can be used for reference. - - Also, each data model in "0000123704_SAFT_Generic_Data_Model.xlsx" file needs to be modified to include an additional column that provides information regarding which commands must be configured for TP function as shown in the screenshot below: - -![Screenshot](Screenshots/tpfunc.png) - +# ORIGINAL README FROM MOXA +# COMPLETELY OUTDATED AS OF 06/2024 +# DOES NOT WORK WITH CURRENT VERSION +# KEPT FOR INFORMATION AND HISTORICAL DATA + +## Moxa AIG-301- Data Path Configuration +## Pre-requisites + + +The following two Excel files from Saft are required + +- [ ] 0000123704_SAFT_Generic_Data_Model.xlsx +- [ ] I-Sight_Project_Communication_Network_Config.xlsx + +Python Script to generate shell sript from Execel files + data_path_config.py + +Note: The python script and the excel files must be kept in the same directory. The files generated by the python script will be present inside the "Generated_Files" folder. +Also, make sure the data provided in the Excel sheet is correct. Else it can affect the configuration process. + +## Step1 + +- [ ] The Python Version used for the project development is: +''' + 3.10.9 +''' + +- [ ] Install all dependencies of the python by running following command: +``` + sudo pip3 install -r requirements.txt +``` + +- [ ] Execute python script in the directory where Execl files are located +``` + python3 data_path_config.py +``` + +OUTPUT: + +On successfull execution of the script, it generates the folder "Generated_Files". The folder contains: + +1. data_path_configuration_shell_script.sh +2. data_config_debug.log + +The "data_path_configuration_shell_script.sh" is the script that needs to be executed in the Moxa device for configuring the ThingsPro Edge software. +The "data_config_debug.log" is the log file generated after running the python script and it performs some basic error checks on the given excel files. + +## Step2 +- [ ] Copy shell script into Moxa gateway in home directory via File Transfer tool example [winscp](https://winscp.net/download/WinSCP-5.21.7-Setup.exe) files + +## Step3 +- [ ] Change file mode of shell script to make it executable by executing the following commands: +``` + sudo sed -i -e 's/\r$//' data_path_configuration_shell_script.sh + sudo chmod +x data_path_configuration_shell_script.sh + +``` +## Step4 +- [ ] Execute shell script in root directory, otheriwse tpfunc will not deployed +``` + sudo su + +``` +``` + ./data_path_configuration_shell_script.sh + +``` +## Step5 + +- [ ] A log report "data_shell_script.log" is generated in the same directory after executing the shell script "data_path_configuration_shell_script.sh". This log report provides information on whether the commands of the shell script were successful or not based on HTTP status codes. Some most common HTTP requests from the log are: + +``` + HTTP request 200 or 201 means that command was successfully executed + HTTP request 400 and above means that the command was NOT executed successfully + +``` +Note: An HTTP request of 400 and above can either mean that the data (which is obtained from the provided excel sheets) within the command is Incorrect or that the ThingsPro Edge software is already configured with that data. + +- [ ] Verify results on ThingsPro Edge webGUI + + 1) Check Modbus Configuration + 2) Check Azure IoT Edge Telemetry + - A seperate topic will be created for each slave + 3) Tpfunc under Function + +## Additional Information + + The "Generated_Files" folder in this repository was created by running the python script with the excel sheets in the same repository. It can be used for reference. + + The Folder "Screenshots" has images that can be used for reference. + + Also, each data model in "0000123704_SAFT_Generic_Data_Model.xlsx" file needs to be modified to include an additional column that provides information regarding which commands must be configured for TP function as shown in the screenshot below: + +![Screenshot](Screenshots/tpfunc.png) + diff --git a/Moulinette/data_path_config.py b/Moulinette/data_path_config.py index 35a4f7d..888d0f4 100644 --- a/Moulinette/data_path_config.py +++ b/Moulinette/data_path_config.py @@ -1,473 +1,467 @@ -import json -import pandas as pd -import inspect -import logging -import string -import os -import shutil -import datetime -import sys -import re -import warnings -import jq -import math - -warnings.simplefilter(action='ignore', category=UserWarning) - -# This dictionary will have tpfunc tags for corresponding slave device to be included in step 4 -tags_of_tpfunc = [] - -# Variables required for TP function -expose_tags = [] -subscribe_tags = [] -assets = [] -assets_name = {} -all_metrics = [] - -jq_filter_set = set() - -allowed_name_characters = list(string.ascii_letters + string.digits) -allowed_name_characters.append('_') -allowed_name_characters.append('-') -allowed_name_characters.append('~') -allowed_name_characters.append('.') - -logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', filename='data_config_debug.log', filemode='w', level=logging.DEBUG, datefmt='%Y%m%d%H%M%S') - -dir_name = 'I-Sight_Generated_Files' -input_datamodel = 'DATAMODEL_1.0.6_MCO2.xlsx' -shell_script_name = dir_name + '/I-Sight_Configuration_' - -if (os.path.isdir(dir_name)): - timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S") - backup_dir_name = f"{dir_name}_backup_{timestamp}" - shutil.move(dir_name, backup_dir_name) - logging.debug("Folder " + dir_name + " already exists, backup made: " + backup_dir_name) -os.mkdir(dir_name) - - -if os.path.isfile(input_datamodel): - logging.debug("Input datamodel " + str(input_datamodel) + " found, archiving it.") - shutil.copy(input_datamodel, dir_name) -else: - print(f"{input_datamodel} not found.") - sys.exit(1) - -def excel_parser(sheet_det, slave_no, slave_device, allowed_name_characters, dsh): - global tags_of_tpfunc - global expose_tags - global subscribe_tags - - df_prop = pd.read_excel(input_datamodel, sheet_name=sheet_det, header = 0) - df_prop_initial_size = len(df_prop) - - df_prop_columns = ["metric_name","modbus_function","modbus_address","type"] - - # The below loop formats the column names properly by replacing the \n with space - for k in range(len(df_prop_columns)): - if '\n' in df_prop_columns[k]: - df_prop_columns[k] = df_prop_columns[k].replace("\n", " ") - df_prop = df_prop.rename(columns = {df_prop.columns[k]:df_prop_columns[k]}) - - dsh.write("# [STEP 2] Assigning " + str(sheet_det) + " Modbus commands to asset " + str(assets_name[slave_no]) + "\n\n") - slave_prop_list = [] - allowed_data_types = ['raw','boolean','int16','int32','int64','uint16','uint32','uint64','float','double','string'] - allowed_data_sizes = [1, 1, 1, 2, 4, 1, 2, 4, 2, 4, 1] - - total_considered_commands = 0 - for index, row in df_prop.iterrows(): - #print(str(index) + " " + str(row)) - filtered_df_prop = df_prop - for k in range(len(df_prop_columns)): - filtered_df_prop = filtered_df_prop[filtered_df_prop[df_prop_columns[k]].notnull()] - logging.debug("Starting registration of Modbus commands for " + str(assets_name[slave_no]) + ".") - for index, row in filtered_df_prop.iterrows(): - #print(str(index) + " " + str(row['metric_name'])) - step2_data = {} - logging.debug("Registering command " + row['metric_name'] + "...") - if allowed_data_sizes[allowed_data_types.index(row['type'])] != 1 and int(row['modbus_quantity']) % allowed_data_sizes[allowed_data_types.index(row['type'])] != 0: - logging.debug("Wrong quantity (" + str(int(row['modbus_quantity'])) + ") for data type " + str(row['type']) + ", using default size " + str(allowed_data_sizes[allowed_data_types.index(row['type'])]) + ".") - step2_data["readQuantity"] = allowed_data_sizes[allowed_data_types.index(row['type'])] - else: - step2_data["readQuantity"] = int(row['modbus_quantity']) - step2_data["remoteDevId"] = slave_no - step2_data["name"] = row['metric_name'] - step2_data["mode"] = 0 - step2_data["func"] = row['modbus_function'] - step2_data["readAddress"] = int(row['modbus_address']) - if not math.isnan(row['poll_interval']): - step2_data["pollInterval"] = int(row['poll_interval']) - else: - logging.debug("Poll interval undefined, using 1000ms as default.") - step2_data["pollInterval"] = 1000 - if row['endian_swap']: - step2_data["swap"] = int(row['endian_swap']) - else: - logging.debug("Endian Swap undefined, not using swap as default.") - step2_data["swap"] = 0 - step2_data["dataType"] = row['type'] - print(row['scaling_factor']) - if not math.isnan(row['scaling_factor']): - step2_data['scalingFunc'] = 1 - step2_data['interceptSlope'] = row['scaling_factor'] - else: - logging.debug("No scaling factor provided, using 1 (no scaling) as default.") - logging.debug(row['metric_name']) - total_considered_commands = total_considered_commands + 1 - print(step2_data) - slave_prop_list.append(step2_data) - all_metrics.append({"remoteDevId": step2_data["remoteDevId"], "metric_name": step2_data["name"], "jqfilter": row['jq_filter_name']}) - logging.debug(str(total_considered_commands) + "/" + str(df_prop_initial_size) + " commands registered.") - if total_considered_commands < df_prop_initial_size: - logging.debug("A command may not be registered if its row is incomplete.") - - df_bitfield = pd.read_excel(input_datamodel, sheet_name='BITFIELDS', header=0) - df_bitfield_columns = ["metric_name","modbus_function","modbus_address"] - filtered_df_bitfield= df_bitfield - for index, row in filtered_df_bitfield.iterrows(): - for column in df_bitfield_columns: - if not check_string_format(column): - filtered_df_bitfield = filtered_df_bitfield[filtered_df_bitfield[column].notnull()] - for index, row in filtered_df_bitfield.iterrows(): - if row['asset_type'] == sheet_det: - logging.debug("Bitfield " + row['metric_name'] + " assigned to " + str(assets_name[slave_no]) + ".") - step2_data = {} - step2_data["remoteDevId"] = slave_no - step2_data["name"] = row['metric_name'] - step2_data["mode"] = 0 - step2_data["func"] = row['modbus_function'] - step2_data["readAddress"] = int(row['modbus_address']) - if row['modbus_quantity']: - step2_data["readQuantity"] = int(row['modbus_quantity']) - else: - step2_data["readQuantity"] = 1 - if row['poll_interval']: - step2_data["pollInterval"] = int(row['poll_interval']) - else: - step2_data["pollInterval"] = 1000 - if row['endian_swap']: - step2_data["swap"] = int(row['endian_swap']) - else: - step2_data["swap"] = 0 - step2_data["dataType"] = 'uint16' - slave_prop_list.append(step2_data) - all_metrics.append({"remoteDevId": step2_data["remoteDevId"], "metric_name": step2_data["name"], "jqfilter": row['jq_filter_name']}) - - slave_prop_list = json.dumps(slave_prop_list) - dsh.write( - inspect.cleandoc( - """sudo curl -X POST https://127.0.0.1:8443/api/v1/modbusmaster/config/mcmds?autoCreate=tags \\ - -H "Content-Type: application/json" \\ - -H "mx-api-token:$(sudo cat /var/thingspro/data/mx-api-token)" \\ - -d """) + "'" + str(slave_prop_list) +"'" + """ -k | jq - """ - ) - dsh.write('printf "\\n \\n" >> data_shell_script.log\n') - dsh.write("\n\n") - - -def common_code(dsh): - dsh.write("# [STEP 3] Applying Modbus configuration\n\n") - dsh.write( - inspect.cleandoc( - """sudo curl -X PUT https://127.0.0.1:8443/api/v1/modbusmaster/control/config/apply \\ - -H "Content-Type: application/json" \\ - -H "mx-api-token:$(sudo cat /var/thingspro/data/mx-api-token)" \\ - -d """) + "'" + "'" + """ -k | jq - \n""" - ) - dsh.write('printf "\\n \\n" >> data_shell_script.log\n') - dsh.write("\n\n") - -def jq_filter(current_device, dsh): - - df_validation = pd.read_excel(input_datamodel, sheet_name='VALIDATION', header = 0) - filtered_df_validation = df_validation[df_validation['jq_filter_name'].notnull()] - filtered_df_validation = filtered_df_validation[filtered_df_validation['jq_filter'].notnull()] - for index,row in filtered_df_validation.iterrows(): - filter = row['jq_filter_name'] - if filter in jq_filter_set and filter != "bitfield": - logging.debug("Creating standard telemetry topic " + filter + ".") - jq_data = {} - jq_data["enable"] = True - jq_data["properties"] = [{"key": "deviceType", "value": "AC_GATEWAY"}, {"key": "cdid", "value": current_device}] - jq_data["outputTopic"] = filter - if row['message_polling_interval']: - jq_data["pollingInterval"] = int(row['message_polling_interval']) - else: - jq_data["pollingInterval"] = int(1) - if row['message_time_limit'] and row['message_size_limit']: - jq_data["sendOutThreshold"] = {"size": int(row['message_size_limit']), "time": int(row['message_time_limit'])} - elif not row['message_time_limit'] and row['message_size_limit']: - jq_data["sendOutThreshold"] = {"size": int(row['message_size_limit']), "time": int(30)} - elif row['message_time_limit'] and not row['message_size_limit']: - jq_data["sendOutThreshold"] = {"size": int(256000), "time": int(row['message_time_limit'])} - else: - jq_data["sendOutThreshold"] = {"size": int(256000), "time": int(30)} - final_filter = row["jq_filter"].replace("***device_name***", current_device) - cmd_list = {} - for metric in all_metrics: - if metric["jqfilter"] == filter: - if assets_name[metric["remoteDevId"]] not in cmd_list: - cmd_list[assets_name[metric["remoteDevId"]]] = [] - - cmd_list[assets_name[metric["remoteDevId"]]].append(metric["metric_name"]) - - jq_data["format"] = final_filter - jq_data["tags"]= {"modbus_tcp_master": cmd_list} - json_object = json.dumps(jq_data) - dsh.write("# [STEP 4] Creating " + filter + " standard Azure telemetry topic\n\n") - dsh.write( - inspect.cleandoc( - """sudo curl -X POST https://127.0.0.1:8443/api/v1/azure-iotedge/messages \\ - -H "Content-Type: application/json" \\ - -H "mx-api-token:$(sudo cat /var/thingspro/data/mx-api-token)" \\ - -d """) + "'" + str(json_object) +"'" + """ -k | jq - \n""" - ) - dsh.write('printf "\\n \\n" >> data_shell_script.log\n') - dsh.write("\n\n") - -STATIC_JQ_FILTER_EMISSIONDATE = "(now|todateiso8601)" - -def bitfield_jq_filter(current_device): - - STATIC_JQ_FILTER_EMISSIONDATE = "(now|todateiso8601)" - STATIC_JQ_FILTER_BITNAME = [] - STATIC_JQ_FILTER_BITSOURCE = [] - STATIC_JQ_FILTER_BITFORMULA = [] - STATIC_JQ_FILTER_TIMESTAMP = "((.ts/1000000)|todateiso8601)" - - df_bitfield = pd.read_excel(input_datamodel, sheet_name='BITFIELDS', header=0) - for i,r in df_bitfield.iterrows(): - for metric in all_metrics: - if metric["metric_name"] == r["metric_name"]: - logging.debug("Creating bitfield specific telemetry topic" + metric['metric_name'] + " for " + str(assets_name[metric["remoteDevId"]]) + ".") - final_filter = {} - json_request = {} - json_request["enable"] = True - json_request["properties"] = [{"key": "deviceType", "value": "AC_GATEWAY"}, {"key": "cdid", "value": current_device}] - json_request["outputTopic"] = metric["metric_name"] - json_request["pollingInterval"] = 60 - json_request["sendOutThreshold"] = {"size": 256000, "time": 180} - json_format = {} - json_format["version"] = 3 - json_format["deviceId"] = current_device - json_format["emissionDate"] = "PLACEHOLDER_EMISSIONDATE" - json_format["deviceType"] = "AC_GATEWAY" - json_format["faults"] = [] - for i in range(1, 17): - current_power = pow(2, i-1) - current_name = "bit_name_" + str(i) - current_source = "bit_source_" + str(i) - current_formula = "((.dataValue)/" + str(current_power) + "%2)" - current_fault = {"name": "PLACEHOLDER_NAME", "idAsset": "PLACEHOLDER_SOURCE", "active": "PLACEHOLDER_FORMULA", "timestamp": "PLACEHOLDER_TIMESTAMP"} - if not pd.isna(r[current_name]) and not pd.isna(r[current_source]): - json_format["faults"].append(current_fault) - STATIC_JQ_FILTER_BITNAME.append(r[current_name]) - STATIC_JQ_FILTER_BITSOURCE.append(r[current_source]) - STATIC_JQ_FILTER_BITFORMULA.append(current_formula) - elif not pd.isna(r[current_name]) and pd.isna(r[current_source]): - json_format["faults"].append(current_fault) - STATIC_JQ_FILTER_BITNAME.append(r[current_name]) - STATIC_JQ_FILTER_BITSOURCE.append(str(assets_name[metric["remoteDevId"]])) - STATIC_JQ_FILTER_BITFORMULA.append(current_formula) - json_request["format"] = json.dumps(json_format) - json_request["tags"]= {} - json_request["tags"]["modbus_tcp_master"] = {} - json_request["tags"]["modbus_tcp_master"][assets_name[metric["remoteDevId"]]] = [metric["metric_name"]] - json_request_dump = json.dumps(json_request) - json_request_dump = json_request_dump.replace("\\\"PLACEHOLDER_EMISSIONDATE\\\"", STATIC_JQ_FILTER_EMISSIONDATE) - for bit in range(len(STATIC_JQ_FILTER_BITNAME)): - json_request_dump = json_request_dump.replace("PLACEHOLDER_NAME", STATIC_JQ_FILTER_BITNAME[bit], 1) - json_request_dump = json_request_dump.replace("PLACEHOLDER_SOURCE", STATIC_JQ_FILTER_BITSOURCE[bit], 1) - json_request_dump = json_request_dump.replace("\\\"PLACEHOLDER_FORMULA\\\"", STATIC_JQ_FILTER_BITFORMULA[bit], 1) - json_request_dump = json_request_dump.replace("\\\"PLACEHOLDER_TIMESTAMP\\\"", STATIC_JQ_FILTER_TIMESTAMP, 1) - dsh.write("# [STEP 5] Creating " + metric['metric_name'] + " bitfield specific Azure telemetry topic\n\n") - logging.debug(json_request_dump) - dsh.write( - inspect.cleandoc( - """sudo curl -X POST https://127.0.0.1:8443/api/v1/azure-iotedge/messages \\ - -H "Content-Type: application/json" \\ - -H "mx-api-token:$(sudo cat /var/thingspro/data/mx-api-token)" \\ - -d """) + "'" + json_request_dump +"'" + """ -k | jq - \n""" - ) - dsh.write('printf "\\n \\n" >> data_shell_script.log\n') - dsh.write("\n\n") - - - -def slave_script(slave_no, row, dsh): - - slave_device = "" - for k in row['asset_name']: - if k in allowed_name_characters: - slave_device = slave_device +k - - slave_ip = row['ip_address'] - slave_port = row['port'] - slave_id = row['slave_id'] - assets_name[slave_no] = slave_device - step1_dict = {"masterTcpIfaceId": 1,"name": slave_device,"enable": 1,"slaveIpaddr": slave_ip,"slaveTcpPort": slave_port, "slaveId": slave_id, "remoteDevId": slave_no} - json_object = json.dumps(step1_dict) - dsh.write("# [STEP 1] Creating asset "+ slave_device + "\n\n") - dsh.write( - inspect.cleandoc( - """sudo curl -X POST https://127.0.0.1:8443/api/v1/modbusmaster/config/remote-devs \\ - -H "Content-Type: application/json" \\ - -H "mx-api-token:$(sudo cat /var/thingspro/data/mx-api-token)" \\ - -d """) + "'" + str(json_object) +"'" + """ -k | jq - \n""" - ) - dsh.write('printf "\\n \\n" >> data_shell_script.log\n') - dsh.write("\n\n") - - # Add the contents related to each slave device - sheet_det = row['asset_type'] - logging.debug("Asset " + slave_device + " created!") - - # The below function retrieves the data required for step 2 of shell script - excel_parser(sheet_det, slave_no, slave_device, allowed_name_characters, dsh) - -def tpfunc_gen(): - global expose_tags - global subscribe_tags - - package = {} - package["name"] = "demoDataFunc" - package["enabled"] = True - package["trigger"] = {"driven":"timeDriven","timeDriven":{"mode":"boot","intervalSec":2,"cronJob":""}} - package["expose"] = {"tags": expose_tags} - package["executable"] = {"language":"python"} - package["params"] = {"setting":{"tpe_publish_interval":1,"test_mode":False},"subscribeTags":subscribe_tags} - try: - package = json.dumps(package) - except Exception as e: - print(f"The exception here is {e}") - - dsh = open(shell_script_name_final,'a') - dsh.write("\n# Shell Code to create TP Function: \n\n") - cmd1 = """echo '#!/usr/bin/python\n# -*- coding: utf-8 -*-\n\nfrom thingspro.edge.func_v1 import package\nfrom thingspro.edge.tag_v1 import tag as tpeTAG\nfrom collections import deque\nimport time\nimport logging\n\n__author__ = "Moxa Europe"\n__license__ = "MIT"\n__version__ = "0.0.1"\n__status__ = "beta"\n\nlog_format = "%(asctime)s: %(levelname)s - %(name)s - %(message)s"\nlogging.basicConfig(level=logging.INFO, datefmt="[%Y-%m-%d %H:%M:%S]", format=log_format)\nlogger = logging.getLogger(__name__)\n\n\n\ncache = {}\n\nclass TpeSaftContext():\n def __init__(self):\n \n # initialize app default settings \n self._tpe_publish_interval = 1\n self._test_mode = False\n \n self._publisher = None\n self._subscriber = None \n \n self._vtag_tags = 0 \n self._vtag_publish_data = deque(maxlen=100)\n self._tagList = [] \n \n\n # initialize virtual tags \n self.tagValueInputVoltage = 0\n self.tagValueInputCurrent = 0\n \n # create config instance to read parameters from package.json file \n config = package.Configuration()\n self._params = config.parameters()\n \n # create subscriber client instance\n self._subscriber = tpeTAG.Subscriber()\n self._subscriber.subscribe_callback(self.callback)\n \n def parse_configuration(self):\n print("*********** Parse Configuration ***********")\n # create config instance to read parameters from package.json file \n \n if "setting" in self._params:\n if "tpe_publish_interval" in self._params["setting"]:\n self._tpe_publish_interval = self._params["setting"]["tpe_publish_interval"]\n print("tpe_publish_interval : " + str(self._tpe_publish_interval))\n \n if "test_mode" in self._params["setting"]:\n self._test_mode = self._params["setting"]["test_mode"]\n print("test_mode : " + str(self._test_mode))\n \n if "subscribeTags" in self._params:\n self._tagList = self._params["subscribeTags"]\n print("subscribeTags: ", self._params["subscribeTags"])\n \n print("**** Parse Configuration Successfull! ****")\n return\n \n \n def _callback(self, data={}):\n #print("************** callback function is called *****************")\n \n # Get tag names from package.json file\n for tagDict in self._tagList:\n print(tagDict)\n if "tagName" in tagDict:\n tagName = tagDict["tagName"]\n print(tagName)\n \n # Compare tagHub tagName with tagName comes from package.json file.\n if data["tagName"] == tagName:\n self.put_to_publish_queue("virtual", data["srcName"]+"_onChange", data["tagName"], data["dataValue"], "double", data["ts"]) \n\n return\n \n \n def callback(self, data={}):\n #print("************** callback function is called *****************")\n \n # Get tag names from package.json file\n for tagDict in self._tagList:\n if "tagName" in tagDict:\n tagName = tagDict["tagName"]\n \n # Compare tagHub tagName with tagName comes from package.json file.\n if data["tagName"] == tagName:\n global cache \n # The following store distint tagName in cache\n if tagName not in cache.keys():\n cache[tagName] = data["dataValue"] \n print("CACHE:", cache) \n self.put_to_publish_queue("virtual", data["srcName"]+"_onChange", data["tagName"], data["dataValue"], "double", data["ts"]) \n print("Cache value of virtual/onChange/" + str(tagName) + " :" + str(cache[tagName])) \n else:\n if cache[tagName] == data["dataValue"]:\n print("No Changes in " + tagName + ": ", data["dataValue"])\n else:\n print("Updated virtual/onChange/" + str(tagName) + " by new vlaue: " + str(data["dataValue"])) \n self.put_to_publish_queue("virtual", data["srcName"]+"_onChange", data["tagName"], data["dataValue"], "double", data["ts"]) \n # update cache tagValue with new value\n cache[tagName] = data["dataValue"]\n print("CACHE:", cache)\n \n return\n \n def register_tpe_callback(self):\n # create subscriber client instance\n self._subscriber = tpeTAG.Subscriber()\n self._subscriber.subscribe_callback(self.callback)\n \n def subscribe_tag(self):\n #print("**************** subscribe_tag function is called ***************")\n \n if "subscribeTags" in self._params:\n tags = self._params["subscribeTags"]\n #print("[Subscribe]:", tags)\n for tag in tags:\n try:\n self._subscriber.subscribe(tag["prvdName"], tag["srcName"], [tag["tagName"]])\n except ValueError:\n pass\n if self._test_mode :\n self._subscriber.subscribe("system", "status", ["cpuUsage"])\n return\n \n \n def put_to_publish_queue(self, prvdName, srcName, tagName, dataValue, dataType, timestamp):\n #print("****************** put_to_publish_queue function is called *********************")\n tag = {\n "prvdName": prvdName,\n "srcName": srcName,\n "tagName": tagName, \n "dataValue": dataValue,\n "dataType" : dataType,\n "ts": timestamp\n } \n self._vtag_publish_data.append(tag)\n return True\n \n \n def tpe_publish(self): \n #print("************** tpe_publish function is called *****************")\n \n #self.subscribe_tag()\n \n # print("Length _vtag_publish_data", len(self._vtag_publish_data))\n while len(self._vtag_publish_data)>0:\n tag = self._vtag_publish_data.popleft()\n self._publisher.publish(tag)\n print("[Publish]:", tag)\n #print("publish: " + tag["tagName"] + ":" + str(tag["dataValue"]) )\n return\n \n \n \nif __name__ == "__main__":\n\n my_app = TpeSaftContext()\n \n # initial configuration from package.json file\n my_app.parse_configuration()\n\n # subscribe Tags\n my_app.subscribe_tag()\n \n # create publisher client instance\n my_app._publisher = tpeTAG.Publisher()\n \n # create direct access instance\n # my_app._accesser = tpeTAG.Access()\n \n \n # infinite loop\n while True: \n my_app.tpe_publish()\n print("sleep " + str(my_app._tpe_publish_interval))\n time.sleep(my_app._tpe_publish_interval)' > demoDataFunc/index.py""" - cmd1 = cmd1.replace("\n","\\n") - dsh.write(inspect.cleandoc("""mkdir demoDataFunc""")) - dsh.write('\n') - dsh.write(cmd1) - dsh.write('\n') - dsh.write("""echo '""" + str(package) + "'" + " | jq '.'" + " > demoDataFunc/package.json") - dsh.write('\n') - dsh.write(inspect.cleandoc("""tpfunc add demoDataFunc""")) - dsh.close() - - # open both files - with open(shell_script_name,'r') as firstfile, open(shell_script_name_final,'a') as secondfile: - # read content from first file - for line in firstfile: - # append content to second file - secondfile.write(line) - os.remove(shell_script_name) - -def check_string_format(input_string): - # define the pattern using regular expression - pattern = r'^bit_(name|source)_(1[0-6]|[1-9])$' - # check if the input string matches the pattern - match = re.match(pattern, input_string) - # return True if there is a match, False otherwise - return bool(match) - -def main(): - # Create the shell script to write content - global dsh - assets = [] - - # Read the VLAN_Modbus spreadsheet from the "I-Sight_Project_Communication_Network_Config.xlsx" file - df_slave = pd.read_excel(input_datamodel, sheet_name='DEVICES', header = 0) - print(df_slave) - - # # The below loop formats the column names properly by replacing the \n with space - df_slave_columns = list(df_slave.columns) - for k in range(len(df_slave_columns)): - if '\n' in df_slave_columns[k]: - df_slave_columns[k] = df_slave_columns[k].replace("\n", " ") - df_slave = df_slave.rename(columns = {df_slave.columns[k]:df_slave_columns[k]}) - - - - null_elements_1 = list(df_slave['device_name'].notnull()) - null_elements_2 = list(df_slave['device_ip_address_http'].notnull()) - for index in range(len(null_elements_1)): - try: - if null_elements_1[index] == False: - if null_elements_2[index] == False: - logging.debug("The slave device %s at index %d is not considered due to missing data model and missing IP address \n",df_slave.at[index,'Equipment Designation'],index+2) - else: - logging.debug("The slave device %s at index %d is not considered due to missing data model \n",df_slave.at[index,'Equipment Designation'],index+2) - else: - if null_elements_2[index] == False: - logging.debug("The slave device %s at index %d is not considered due to missing IP address \n",df_slave.at[index,'Equipment Designation'],index+2) - except Exception as e: - print(e) - - filtered_df_slave = df_slave[df_slave['device_name'].notnull()] - filtered_df_slave = filtered_df_slave[filtered_df_slave['device_ip_address_http'].notnull()] - - slave_no = 1 - - current_device = "" - - for index, row in filtered_df_slave.iterrows(): - current_device = str(row['device_name']) - logging.debug("Defining parameters for " + current_device + " Moxa device...") - dsh = open (shell_script_name + current_device + '.sh', 'w') - df_assets = pd.read_excel(input_datamodel, sheet_name='ASSETS', header = 0) - df_assets_columns = ["asset_name", "asset_type", "ip_address", "port", "device"] - filtered_df_assets = df_assets - - for k in range(len(df_assets_columns)): - filtered_df_assets = filtered_df_assets[filtered_df_assets[df_assets_columns[k]].notnull()] - filtered_df_assets = filtered_df_assets[filtered_df_assets['device'] == row['device_name']] - filtered_df_assets.drop_duplicates(subset=['asset_type', 'ip_address', 'slave_id', 'port', 'device'], inplace=True) - - for index, row in filtered_df_assets.iterrows(): - current_asset = {"ip_address": row['ip_address'], "slave_id": row['slave_id'], "port": row['port']} - exists = False - existing = 0 - for a in range(len(assets)): - if current_asset == assets[a]: - exists = True - existing = a + 1 - logging.debug("Asset " + row['asset_name'] + " not created because it already exists. Processing it under the existing asset.") - excel_parser(row['asset_type'], existing, row["asset_name"], allowed_name_characters, dsh) - break - - if exists != True: - assets.append(current_asset) - logging.debug("Creating new asset...") - slave_script(slave_no, row, dsh) - slave_no = slave_no + 1 - - common_code(dsh) - - for metric in range(len(all_metrics)): - jq_filter_set.add(all_metrics[metric]['jqfilter']) - jq_filter(current_device, dsh) - logging.debug("BITFIELDS STARTED") - bitfield_jq_filter(current_device) - logging.debug("BITDIELDS ENDED") - logging.debug("Parameters for " + current_device + " Moxa device completed!") - - - all_metrics.clear() - #bitfields(dsh, filtered_df_assets, current_device, assets) - dsh.write("# [STEP 6] Ending configuration.") - dsh.close() - - - - -if __name__ == "__main__": - logging.debug("Process started...") - main() - logging.shutdown() - shutil.move("data_config_debug.log", dir_name + "/data_config_debug.log") +import json +import pandas as pd +import inspect +import logging +import string +import os +import shutil +import datetime +import sys +import re +import warnings +import jq +import math + +warnings.simplefilter(action='ignore', category=UserWarning) + +# This dictionary will have tpfunc tags for corresponding slave device to be included in step 4 +tags_of_tpfunc = [] + +# Variables required for TP function +expose_tags = [] +subscribe_tags = [] +assets = [] +assets_name = {} +all_metrics = [] + +jq_filter_set = set() + +allowed_name_characters = list(string.ascii_letters + string.digits) +allowed_name_characters.append('_') +allowed_name_characters.append('-') +allowed_name_characters.append('~') +allowed_name_characters.append('.') + +logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', filename='data_config_debug.log', filemode='w', level=logging.DEBUG, datefmt='%Y%m%d%H%M%S') + +dir_name = 'I-Sight_Generated_Files' +input_datamodel = 'DATAMODEL_1.0.6_SASK.xlsx' +shell_script_name = dir_name + '/I-Sight_Configuration_' + +if (os.path.isdir(dir_name)): + timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S") + backup_dir_name = f"{dir_name}_backup_{timestamp}" + shutil.move(dir_name, backup_dir_name) + logging.debug("Folder " + dir_name + " already exists, backup made: " + backup_dir_name) +os.mkdir(dir_name) + + +if os.path.isfile(input_datamodel): + logging.debug("Input datamodel " + str(input_datamodel) + " found, archiving it.") + shutil.copy(input_datamodel, dir_name) +else: + print(f"{input_datamodel} not found.") + sys.exit(1) + +def excel_parser(sheet_det, slave_no, slave_device, allowed_name_characters, dsh): + global tags_of_tpfunc + global expose_tags + global subscribe_tags + + df_prop = pd.read_excel(input_datamodel, sheet_name=sheet_det, header = 0) + df_prop_initial_size = len(df_prop) + + df_prop_columns = ["metric_name","modbus_function","modbus_address","type"] + + # The below loop formats the column names properly by replacing the \n with space + for k in range(len(df_prop_columns)): + if '\n' in df_prop_columns[k]: + df_prop_columns[k] = df_prop_columns[k].replace("\n", " ") + df_prop = df_prop.rename(columns = {df_prop.columns[k]:df_prop_columns[k]}) + + dsh.write("# [STEP 2] Assigning " + str(sheet_det) + " Modbus commands to asset " + str(assets_name[slave_no]) + "\n\n") + slave_prop_list = [] + allowed_data_types = ['raw','boolean','int16','int32','int64','uint16','uint32','uint64','float','double','string'] + allowed_data_sizes = [1, 1, 1, 2, 4, 1, 2, 4, 2, 4, 1] + + total_considered_commands = 0 + for index, row in df_prop.iterrows(): + #print(str(index) + " " + str(row)) + filtered_df_prop = df_prop + for k in range(len(df_prop_columns)): + filtered_df_prop = filtered_df_prop[filtered_df_prop[df_prop_columns[k]].notnull()] + logging.debug("Starting registration of Modbus commands for " + str(assets_name[slave_no]) + ".") + for index, row in filtered_df_prop.iterrows(): + #print(str(index) + " " + str(row['metric_name'])) + step2_data = {} + logging.debug("Registering command " + row['metric_name'] + "...") + if allowed_data_sizes[allowed_data_types.index(row['type'])] != 1 and int(row['modbus_quantity']) % allowed_data_sizes[allowed_data_types.index(row['type'])] != 0: + logging.debug("Wrong quantity (" + str(int(row['modbus_quantity'])) + ") for data type " + str(row['type']) + ", using default size " + str(allowed_data_sizes[allowed_data_types.index(row['type'])]) + ".") + step2_data["readQuantity"] = allowed_data_sizes[allowed_data_types.index(row['type'])] + else: + step2_data["readQuantity"] = int(row['modbus_quantity']) + step2_data["remoteDevId"] = slave_no + step2_data["name"] = row['metric_name'] + step2_data["mode"] = 0 + step2_data["func"] = row['modbus_function'] + step2_data["readAddress"] = int(row['modbus_address']) + if not math.isnan(row['poll_interval']): + step2_data["pollInterval"] = int(row['poll_interval']) + else: + logging.debug("Poll interval undefined, using 1000ms as default.") + step2_data["pollInterval"] = 1000 + if row['endian_swap']: + step2_data["swap"] = int(row['endian_swap']) + else: + logging.debug("Endian Swap undefined, not using swap as default.") + step2_data["swap"] = 0 + step2_data["dataType"] = row['type'] + print(row['scaling_factor']) + if not math.isnan(row['scaling_factor']): + step2_data['scalingFunc'] = 1 + step2_data['interceptSlope'] = row['scaling_factor'] + else: + logging.debug("No scaling factor provided, using 1 (no scaling) as default.") + logging.debug(row['metric_name']) + total_considered_commands = total_considered_commands + 1 + print(step2_data) + slave_prop_list.append(step2_data) + all_metrics.append({"remoteDevId": step2_data["remoteDevId"], "metric_name": step2_data["name"], "jqfilter": row['jq_filter_name']}) + logging.debug(str(total_considered_commands) + "/" + str(df_prop_initial_size) + " commands registered.") + if total_considered_commands < df_prop_initial_size: + logging.debug("A command may not be registered if its row is incomplete.") + + df_bitfield = pd.read_excel(input_datamodel, sheet_name='BITFIELDS', header=0) + df_bitfield_columns = ["metric_name","modbus_function","modbus_address"] + filtered_df_bitfield= df_bitfield + for index, row in filtered_df_bitfield.iterrows(): + for column in df_bitfield_columns: + if not check_string_format(column): + filtered_df_bitfield = filtered_df_bitfield[filtered_df_bitfield[column].notnull()] + for index, row in filtered_df_bitfield.iterrows(): + if row['asset_type'] == sheet_det: + logging.debug("Bitfield " + row['metric_name'] + " assigned to " + str(assets_name[slave_no]) + ".") + step2_data = {} + step2_data["remoteDevId"] = slave_no + step2_data["name"] = row['metric_name'] + step2_data["mode"] = 0 + step2_data["func"] = row['modbus_function'] + step2_data["readAddress"] = int(row['modbus_address']) + if row['modbus_quantity']: + step2_data["readQuantity"] = int(row['modbus_quantity']) + else: + step2_data["readQuantity"] = 1 + if row['poll_interval']: + step2_data["pollInterval"] = int(row['poll_interval']) + else: + step2_data["pollInterval"] = 1000 + if row['endian_swap']: + step2_data["swap"] = int(row['endian_swap']) + else: + step2_data["swap"] = 0 + step2_data["dataType"] = 'uint16' + slave_prop_list.append(step2_data) + all_metrics.append({"remoteDevId": step2_data["remoteDevId"], "metric_name": step2_data["name"], "jqfilter": row['jq_filter_name']}) + + slave_prop_list = json.dumps(slave_prop_list) + dsh.write( + inspect.cleandoc( + """sudo curl -X POST https://127.0.0.1:8443/api/v1/modbusmaster/config/mcmds?autoCreate=tags \\ + -H "Content-Type: application/json" \\ + -H "mx-api-token:$(sudo cat /var/thingspro/data/mx-api-token)" \\ + -d """) + "'" + str(slave_prop_list) +"'" + """ -k | jq + """ + ) + dsh.write('printf "\\n \\n" >> data_shell_script.log\n') + dsh.write("\n\n") + + +def common_code(dsh): + dsh.write("# [STEP 3] Applying Modbus configuration\n\n") + dsh.write( + inspect.cleandoc( + """sudo curl -X PUT https://127.0.0.1:8443/api/v1/modbusmaster/control/config/apply \\ + -H "Content-Type: application/json" \\ + -H "mx-api-token:$(sudo cat /var/thingspro/data/mx-api-token)" \\ + -d """) + "'" + "'" + """ -k | jq + \n""" + ) + dsh.write('printf "\\n \\n" >> data_shell_script.log\n') + dsh.write("\n\n") + +def jq_filter(current_device, dsh): + + df_validation = pd.read_excel(input_datamodel, sheet_name='VALIDATION', header = 0) + filtered_df_validation = df_validation[df_validation['jq_filter_name'].notnull()] + filtered_df_validation = filtered_df_validation[filtered_df_validation['jq_filter'].notnull()] + for index,row in filtered_df_validation.iterrows(): + filter = row['jq_filter_name'] + if filter in jq_filter_set and filter != "bitfield": + logging.debug("Creating standard telemetry topic " + filter + ".") + jq_data = {} + jq_data["enable"] = False + jq_data["properties"] = [{"key": "deviceType", "value": "AC_GATEWAY"}, {"key": "cdid", "value": current_device}] + jq_data["outputTopic"] = filter + jq_data["sendOutThreshold"] = {"mode": "bySize", "size": int(128000), "time": int(30), "sizeIdleTimer": {"enable": True, "time": int(30)}} + jq_data["minPublishInterval"] = int(0) + jq_data["samplingMode"] = "allValues" + jq_data["customSamplingRate"] = False + jq_data["pollingInterval"] = int(0) + jq_data["onChange"] = False + final_filter = row["jq_filter"].replace("***device_name***", current_device) + cmd_list = {} + for metric in all_metrics: + if metric["jqfilter"] == filter: + if assets_name[metric["remoteDevId"]] not in cmd_list: + cmd_list[assets_name[metric["remoteDevId"]]] = [] + + cmd_list[assets_name[metric["remoteDevId"]]].append(metric["metric_name"]) + + jq_data["format"] = final_filter + jq_data["tags"]= {"modbus_tcp_master": cmd_list} + json_object = json.dumps(jq_data) + dsh.write("# [STEP 4] Creating " + filter + " standard Azure telemetry topic\n\n") + dsh.write( + inspect.cleandoc( + """sudo curl -X POST https://127.0.0.1:8443/api/v1/azure-iotedge/messages \\ + -H "Content-Type: application/json" \\ + -H "mx-api-token:$(sudo cat /var/thingspro/data/mx-api-token)" \\ + -d """) + "'" + str(json_object) +"'" + """ -k | jq + \n""" + ) + dsh.write('printf "\\n \\n" >> data_shell_script.log\n') + dsh.write("\n\n") + +STATIC_JQ_FILTER_EMISSIONDATE = "(now|todateiso8601)" + +def bitfield_jq_filter(current_device): + + STATIC_JQ_FILTER_EMISSIONDATE = "(now|todateiso8601)" + STATIC_JQ_FILTER_BITNAME = [] + STATIC_JQ_FILTER_BITSOURCE = [] + STATIC_JQ_FILTER_BITFORMULA = [] + STATIC_JQ_FILTER_TIMESTAMP = "((.ts/1000000)|todateiso8601)" + + df_bitfield = pd.read_excel(input_datamodel, sheet_name='BITFIELDS', header=0) + for i,r in df_bitfield.iterrows(): + for metric in all_metrics: + if metric["metric_name"] == r["metric_name"]: + logging.debug("Creating bitfield specific telemetry topic" + metric['metric_name'] + " for " + str(assets_name[metric["remoteDevId"]]) + ".") + final_filter = {} + json_request = {} + json_request["enable"] = False + json_request["properties"] = [{"key": "deviceType", "value": "AC_GATEWAY"}, {"key": "cdid", "value": current_device}] + json_request["outputTopic"] = metric["metric_name"] + json_request["pollingInterval"] = 60 + json_request["sendOutThreshold"] = {"size": 256000, "time": 180} + json_format = {} + json_format["version"] = 3 + json_format["deviceId"] = current_device + json_format["emissionDate"] = "PLACEHOLDER_EMISSIONDATE" + json_format["deviceType"] = "AC_GATEWAY" + json_format["faults"] = [] + for i in range(1, 17): + current_power = pow(2, i-1) + current_name = "bit_name_" + str(i) + current_source = "bit_source_" + str(i) + current_formula = "((.dataValue)/" + str(current_power) + "%2)" + current_fault = {"name": "PLACEHOLDER_NAME", "idAsset": "PLACEHOLDER_SOURCE", "active": "PLACEHOLDER_FORMULA", "timestamp": "PLACEHOLDER_TIMESTAMP"} + if not pd.isna(r[current_name]) and not pd.isna(r[current_source]): + json_format["faults"].append(current_fault) + STATIC_JQ_FILTER_BITNAME.append(r[current_name]) + STATIC_JQ_FILTER_BITSOURCE.append(r[current_source]) + STATIC_JQ_FILTER_BITFORMULA.append(current_formula) + elif not pd.isna(r[current_name]) and pd.isna(r[current_source]): + json_format["faults"].append(current_fault) + STATIC_JQ_FILTER_BITNAME.append(r[current_name]) + STATIC_JQ_FILTER_BITSOURCE.append(str(assets_name[metric["remoteDevId"]])) + STATIC_JQ_FILTER_BITFORMULA.append(current_formula) + json_request["format"] = json.dumps(json_format) + json_request["tags"]= {} + json_request["tags"]["modbus_tcp_master"] = {} + json_request["tags"]["modbus_tcp_master"][assets_name[metric["remoteDevId"]]] = [metric["metric_name"]] + json_request_dump = json.dumps(json_request) + json_request_dump = json_request_dump.replace("\\\"PLACEHOLDER_EMISSIONDATE\\\"", STATIC_JQ_FILTER_EMISSIONDATE) + for bit in range(len(STATIC_JQ_FILTER_BITNAME)): + json_request_dump = json_request_dump.replace("PLACEHOLDER_NAME", STATIC_JQ_FILTER_BITNAME[bit], 1) + json_request_dump = json_request_dump.replace("PLACEHOLDER_SOURCE", STATIC_JQ_FILTER_BITSOURCE[bit], 1) + json_request_dump = json_request_dump.replace("\\\"PLACEHOLDER_FORMULA\\\"", STATIC_JQ_FILTER_BITFORMULA[bit], 1) + json_request_dump = json_request_dump.replace("\\\"PLACEHOLDER_TIMESTAMP\\\"", STATIC_JQ_FILTER_TIMESTAMP, 1) + dsh.write("# [STEP 5] Creating " + metric['metric_name'] + " bitfield specific Azure telemetry topic\n\n") + logging.debug(json_request_dump) + dsh.write( + inspect.cleandoc( + """sudo curl -X POST https://127.0.0.1:8443/api/v1/azure-iotedge/messages \\ + -H "Content-Type: application/json" \\ + -H "mx-api-token:$(sudo cat /var/thingspro/data/mx-api-token)" \\ + -d """) + "'" + json_request_dump +"'" + """ -k | jq + \n""" + ) + dsh.write('printf "\\n \\n" >> data_shell_script.log\n') + dsh.write("\n\n") + + + +def slave_script(slave_no, row, dsh): + + slave_device = "" + for k in row['asset_name']: + if k in allowed_name_characters: + slave_device = slave_device +k + + slave_ip = row['ip_address'] + slave_port = row['port'] + slave_id = row['slave_id'] + assets_name[slave_no] = slave_device + step1_dict = {"masterTcpIfaceId": 1,"name": slave_device,"enable": 1,"slaveIpaddr": slave_ip,"slaveTcpPort": slave_port, "slaveId": slave_id, "remoteDevId": slave_no} + json_object = json.dumps(step1_dict) + dsh.write("# [STEP 1] Creating asset "+ slave_device + "\n\n") + dsh.write( + inspect.cleandoc( + """sudo curl -X POST https://127.0.0.1:8443/api/v1/modbusmaster/config/remote-devs \\ + -H "Content-Type: application/json" \\ + -H "mx-api-token:$(sudo cat /var/thingspro/data/mx-api-token)" \\ + -d """) + "'" + str(json_object) +"'" + """ -k | jq + \n""" + ) + dsh.write('printf "\\n \\n" >> data_shell_script.log\n') + dsh.write("\n\n") + + # Add the contents related to each slave device + sheet_det = row['asset_type'] + logging.debug("Asset " + slave_device + " created!") + + # The below function retrieves the data required for step 2 of shell script + excel_parser(sheet_det, slave_no, slave_device, allowed_name_characters, dsh) + +def tpfunc_gen(): + global expose_tags + global subscribe_tags + + package = {} + package["name"] = "demoDataFunc" + package["enabled"] = False + package["trigger"] = {"driven":"timeDriven","timeDriven":{"mode":"boot","intervalSec":2,"cronJob":""}} + package["expose"] = {"tags": expose_tags} + package["executable"] = {"language":"python"} + package["params"] = {"setting":{"tpe_publish_interval":1,"test_mode":False},"subscribeTags":subscribe_tags} + try: + package = json.dumps(package) + except Exception as e: + print(f"The exception here is {e}") + + dsh = open(shell_script_name_final,'a') + dsh.write("\n# Shell Code to create TP Function: \n\n") + cmd1 = """echo '#!/usr/bin/python\n# -*- coding: utf-8 -*-\n\nfrom thingspro.edge.func_v1 import package\nfrom thingspro.edge.tag_v1 import tag as tpeTAG\nfrom collections import deque\nimport time\nimport logging\n\n__author__ = "Moxa Europe"\n__license__ = "MIT"\n__version__ = "0.0.1"\n__status__ = "beta"\n\nlog_format = "%(asctime)s: %(levelname)s - %(name)s - %(message)s"\nlogging.basicConfig(level=logging.INFO, datefmt="[%Y-%m-%d %H:%M:%S]", format=log_format)\nlogger = logging.getLogger(__name__)\n\n\n\ncache = {}\n\nclass TpeSaftContext():\n def __init__(self):\n \n # initialize app default settings \n self._tpe_publish_interval = 1\n self._test_mode = False\n \n self._publisher = None\n self._subscriber = None \n \n self._vtag_tags = 0 \n self._vtag_publish_data = deque(maxlen=100)\n self._tagList = [] \n \n\n # initialize virtual tags \n self.tagValueInputVoltage = 0\n self.tagValueInputCurrent = 0\n \n # create config instance to read parameters from package.json file \n config = package.Configuration()\n self._params = config.parameters()\n \n # create subscriber client instance\n self._subscriber = tpeTAG.Subscriber()\n self._subscriber.subscribe_callback(self.callback)\n \n def parse_configuration(self):\n print("*********** Parse Configuration ***********")\n # create config instance to read parameters from package.json file \n \n if "setting" in self._params:\n if "tpe_publish_interval" in self._params["setting"]:\n self._tpe_publish_interval = self._params["setting"]["tpe_publish_interval"]\n print("tpe_publish_interval : " + str(self._tpe_publish_interval))\n \n if "test_mode" in self._params["setting"]:\n self._test_mode = self._params["setting"]["test_mode"]\n print("test_mode : " + str(self._test_mode))\n \n if "subscribeTags" in self._params:\n self._tagList = self._params["subscribeTags"]\n print("subscribeTags: ", self._params["subscribeTags"])\n \n print("**** Parse Configuration Successfull! ****")\n return\n \n \n def _callback(self, data={}):\n #print("************** callback function is called *****************")\n \n # Get tag names from package.json file\n for tagDict in self._tagList:\n print(tagDict)\n if "tagName" in tagDict:\n tagName = tagDict["tagName"]\n print(tagName)\n \n # Compare tagHub tagName with tagName comes from package.json file.\n if data["tagName"] == tagName:\n self.put_to_publish_queue("virtual", data["srcName"]+"_onChange", data["tagName"], data["dataValue"], "double", data["ts"]) \n\n return\n \n \n def callback(self, data={}):\n #print("************** callback function is called *****************")\n \n # Get tag names from package.json file\n for tagDict in self._tagList:\n if "tagName" in tagDict:\n tagName = tagDict["tagName"]\n \n # Compare tagHub tagName with tagName comes from package.json file.\n if data["tagName"] == tagName:\n global cache \n # The following store distint tagName in cache\n if tagName not in cache.keys():\n cache[tagName] = data["dataValue"] \n print("CACHE:", cache) \n self.put_to_publish_queue("virtual", data["srcName"]+"_onChange", data["tagName"], data["dataValue"], "double", data["ts"]) \n print("Cache value of virtual/onChange/" + str(tagName) + " :" + str(cache[tagName])) \n else:\n if cache[tagName] == data["dataValue"]:\n print("No Changes in " + tagName + ": ", data["dataValue"])\n else:\n print("Updated virtual/onChange/" + str(tagName) + " by new vlaue: " + str(data["dataValue"])) \n self.put_to_publish_queue("virtual", data["srcName"]+"_onChange", data["tagName"], data["dataValue"], "double", data["ts"]) \n # update cache tagValue with new value\n cache[tagName] = data["dataValue"]\n print("CACHE:", cache)\n \n return\n \n def register_tpe_callback(self):\n # create subscriber client instance\n self._subscriber = tpeTAG.Subscriber()\n self._subscriber.subscribe_callback(self.callback)\n \n def subscribe_tag(self):\n #print("**************** subscribe_tag function is called ***************")\n \n if "subscribeTags" in self._params:\n tags = self._params["subscribeTags"]\n #print("[Subscribe]:", tags)\n for tag in tags:\n try:\n self._subscriber.subscribe(tag["prvdName"], tag["srcName"], [tag["tagName"]])\n except ValueError:\n pass\n if self._test_mode :\n self._subscriber.subscribe("system", "status", ["cpuUsage"])\n return\n \n \n def put_to_publish_queue(self, prvdName, srcName, tagName, dataValue, dataType, timestamp):\n #print("****************** put_to_publish_queue function is called *********************")\n tag = {\n "prvdName": prvdName,\n "srcName": srcName,\n "tagName": tagName, \n "dataValue": dataValue,\n "dataType" : dataType,\n "ts": timestamp\n } \n self._vtag_publish_data.append(tag)\n return True\n \n \n def tpe_publish(self): \n #print("************** tpe_publish function is called *****************")\n \n #self.subscribe_tag()\n \n # print("Length _vtag_publish_data", len(self._vtag_publish_data))\n while len(self._vtag_publish_data)>0:\n tag = self._vtag_publish_data.popleft()\n self._publisher.publish(tag)\n print("[Publish]:", tag)\n #print("publish: " + tag["tagName"] + ":" + str(tag["dataValue"]) )\n return\n \n \n \nif __name__ == "__main__":\n\n my_app = TpeSaftContext()\n \n # initial configuration from package.json file\n my_app.parse_configuration()\n\n # subscribe Tags\n my_app.subscribe_tag()\n \n # create publisher client instance\n my_app._publisher = tpeTAG.Publisher()\n \n # create direct access instance\n # my_app._accesser = tpeTAG.Access()\n \n \n # infinite loop\n while True: \n my_app.tpe_publish()\n print("sleep " + str(my_app._tpe_publish_interval))\n time.sleep(my_app._tpe_publish_interval)' > demoDataFunc/index.py""" + cmd1 = cmd1.replace("\n","\\n") + dsh.write(inspect.cleandoc("""mkdir demoDataFunc""")) + dsh.write('\n') + dsh.write(cmd1) + dsh.write('\n') + dsh.write("""echo '""" + str(package) + "'" + " | jq '.'" + " > demoDataFunc/package.json") + dsh.write('\n') + dsh.write(inspect.cleandoc("""tpfunc add demoDataFunc""")) + dsh.close() + + # open both files + with open(shell_script_name,'r') as firstfile, open(shell_script_name_final,'a') as secondfile: + # read content from first file + for line in firstfile: + # append content to second file + secondfile.write(line) + os.remove(shell_script_name) + +def check_string_format(input_string): + # define the pattern using regular expression + pattern = r'^bit_(name|source)_(1[0-6]|[1-9])$' + # check if the input string matches the pattern + match = re.match(pattern, input_string) + # return True if there is a match, False otherwise + return bool(match) + +def main(): + # Create the shell script to write content + global dsh + assets = [] + + # Read the VLAN_Modbus spreadsheet from the "I-Sight_Project_Communication_Network_Config.xlsx" file + df_slave = pd.read_excel(input_datamodel, sheet_name='DEVICES', header = 0) + print(df_slave) + + # # The below loop formats the column names properly by replacing the \n with space + df_slave_columns = list(df_slave.columns) + for k in range(len(df_slave_columns)): + if '\n' in df_slave_columns[k]: + df_slave_columns[k] = df_slave_columns[k].replace("\n", " ") + df_slave = df_slave.rename(columns = {df_slave.columns[k]:df_slave_columns[k]}) + + + + null_elements_1 = list(df_slave['device_name'].notnull()) + null_elements_2 = list(df_slave['device_ip_address_http'].notnull()) + for index in range(len(null_elements_1)): + try: + if null_elements_1[index] == False: + if null_elements_2[index] == False: + logging.debug("The slave device %s at index %d is not considered due to missing data model and missing IP address \n",df_slave.at[index,'Equipment Designation'],index+2) + else: + logging.debug("The slave device %s at index %d is not considered due to missing data model \n",df_slave.at[index,'Equipment Designation'],index+2) + else: + if null_elements_2[index] == False: + logging.debug("The slave device %s at index %d is not considered due to missing IP address \n",df_slave.at[index,'Equipment Designation'],index+2) + except Exception as e: + print(e) + + filtered_df_slave = df_slave[df_slave['device_name'].notnull()] + filtered_df_slave = filtered_df_slave[filtered_df_slave['device_ip_address_http'].notnull()] + + slave_no = 1 + + current_device = "" + + for index, row in filtered_df_slave.iterrows(): + current_device = str(row['device_name']) + logging.debug("Defining parameters for " + current_device + " Moxa device...") + dsh = open (shell_script_name + current_device + '.sh', 'w') + df_assets = pd.read_excel(input_datamodel, sheet_name='ASSETS', header = 0) + df_assets_columns = ["asset_name", "asset_type", "ip_address", "port", "device"] + filtered_df_assets = df_assets + + for k in range(len(df_assets_columns)): + filtered_df_assets = filtered_df_assets[filtered_df_assets[df_assets_columns[k]].notnull()] + filtered_df_assets = filtered_df_assets[filtered_df_assets['device'] == row['device_name']] + filtered_df_assets.drop_duplicates(subset=['asset_type', 'ip_address', 'slave_id', 'port', 'device'], inplace=True) + + for index, row in filtered_df_assets.iterrows(): + current_asset = {"ip_address": row['ip_address'], "slave_id": row['slave_id'], "port": row['port']} + exists = False + existing = 0 + for a in range(len(assets)): + if current_asset == assets[a]: + exists = True + existing = a + 1 + logging.debug("Asset " + row['asset_name'] + " not created because it already exists. Processing it under the existing asset.") + excel_parser(row['asset_type'], existing, row["asset_name"], allowed_name_characters, dsh) + break + + if exists != True: + assets.append(current_asset) + logging.debug("Creating new asset...") + slave_script(slave_no, row, dsh) + slave_no = slave_no + 1 + + common_code(dsh) + + for metric in range(len(all_metrics)): + jq_filter_set.add(all_metrics[metric]['jqfilter']) + jq_filter(current_device, dsh) + logging.debug("BITFIELDS STARTED") + bitfield_jq_filter(current_device) + logging.debug("BITDIELDS ENDED") + logging.debug("Parameters for " + current_device + " Moxa device completed!") + + + all_metrics.clear() + #bitfields(dsh, filtered_df_assets, current_device, assets) + dsh.write("# [STEP 6] Ending configuration.") + dsh.close() + + + + +if __name__ == "__main__": + logging.debug("Process started...") + main() + logging.shutdown() + shutil.move("data_config_debug.log", dir_name + "/data_config_debug.log") \ No newline at end of file diff --git a/Moulinette/requirements.txt b/Moulinette/requirements.txt index 621a976..b0da061 100644 --- a/Moulinette/requirements.txt +++ b/Moulinette/requirements.txt @@ -1,2 +1,2 @@ -pandas==1.4.3 -openpyxl==3.1.0 +pandas==1.4.3 +openpyxl==3.1.0 diff --git a/Python/azure_iot_hub_1.py b/Python/azure_iot_hub_1.py index 76c39e8..bdbca55 100644 --- a/Python/azure_iot_hub_1.py +++ b/Python/azure_iot_hub_1.py @@ -1,45 +1,49 @@ -from azure.iot.hub import IoTHubRegistryManager -from azure.iot.hub.protocol.models import QuerySpecification -from azure.iot.hub.models import CloudToDeviceMethod, CloudToDeviceMethodResult - -import json - -module_id = "thingspro-agent" -method_name = "thingspro-api-v1" -payload = '{"method":"GET", "path":"/device/general"}' - -# Install the Azure IoT Hub SDK: -# pip install azure-iot-hub - -# Authenticate to your Azure account -CONNECTION_STRING = "HostName=IotHub-CUBE-PROD.azure-devices.net;SharedAccessKeyName=iothubowner;SharedAccessKey= ..." -if CONNECTION_STRING == "": - print("Provide a connection string for the Iot Hub before running the script!") - exit(13) - - -registry_manager = IoTHubRegistryManager.from_connection_string(CONNECTION_STRING) -query_spec = QuerySpecification(query="SELECT * FROM devices WHERE tags.site != 'QWE' AND tags.number != '0' AND capabilities.iotEdge = true") - -query_result = registry_manager.query_iot_hub(query_spec) -devices = [] -for item in query_result.items: - devices.append([int(item.tags['number']), item.tags['deviceId'], item.tags['site']]) - -ordered_devices = sorted(devices, key = lambda x: (x[2], x[0])) - -for i in ordered_devices: - current_device_modules = registry_manager.get_modules(i[1]) - for module in current_device_modules: - if module.module_id == module_id: - thingspro_module = module - if thingspro_module: - #print("Found thingspro-agent for " + i[1] + " (" + i[2] + ")") - try: - direct_method = CloudToDeviceMethod(method_name=method_name, payload=json.loads(payload)) - response = registry_manager.invoke_device_module_method(device_id=i[1], module_id=module_id, direct_method_request=direct_method) - print(str(i[2]), str(i[0]), str(i[1]), response.payload['data']['description'],sep=";") - except: - print(str(i[2]), str(i[0]), str(i[1]), "UNREACHABLE",sep=";") - else: +from azure.iot.hub import IoTHubRegistryManager +from azure.iot.hub.protocol.models import QuerySpecification +from azure.iot.hub.models import CloudToDeviceMethod, CloudToDeviceMethodResult + +import json + +module_id = "thingspro-agent" +method_name = "thingspro-api-v1" +payload = '{"method":"GET", "path":"/device/general"}' + +# Install the Azure IoT Hub SDK: +# pip install azure-iot-hub + +# Authenticate to your Azure account +CONNECTION_STRING = "HostName=IotHub-CUBE-PROD.azure-devices.net;SharedAccessKeyName=iothubowner;SharedAccessKey=...=" +if CONNECTION_STRING == "": + print("Provide a connection string for the Iot Hub before running the script!") + exit(13) + + +registry_manager = IoTHubRegistryManager.from_connection_string(CONNECTION_STRING) +query_spec = QuerySpecification(query="SELECT * FROM devices WHERE tags.site = 'MYRTLE' AND tags.number != '0' AND capabilities.iotEdge = true") + +query_result = registry_manager.query_iot_hub(query_spec) +devices = [] +for item in query_result.items: + devices.append([int(item.tags['number']), item.tags['deviceId'], item.tags['site']]) + +ordered_devices = sorted(devices, key = lambda x: (x[2], x[0])) + +for i in ordered_devices: + current_device_modules = registry_manager.get_modules(i[1]) + for module in current_device_modules: + if module.module_id == module_id: + thingspro_module = module + if thingspro_module: + #print("Found thingspro-agent for " + i[1] + " (" + i[2] + ")") + try: + direct_method = CloudToDeviceMethod(method_name=method_name, payload=json.loads(payload)) + response = registry_manager.invoke_device_module_method(device_id=i[1], module_id=module_id, direct_method_request=direct_method) + #print(response.payload) + if str(i[1]) == str(response.payload['data']['hostName']): + print(str(i[2]), str(i[0]), str(i[1]), response.payload['data']['description'], response.payload['data']['firmwareVersion'], sep=";") + else: + print(str(i[2]), str(i[0]), str(i[1]), response.payload['data']['description'], response.payload['data']['hostName'], response.payload['data']['firmwareVersion'], sep=";") + except: + print(str(i[2]), str(i[0]), str(i[1]), "UNREACHABLE",sep=";") + else: print("No thingspro-agent available for " + i[1] + " (" + i[2] + ")") \ No newline at end of file diff --git a/Python/danish_D1_api.py b/Python/danish_D1_api.py index 1dbc418..d3d1451 100644 --- a/Python/danish_D1_api.py +++ b/Python/danish_D1_api.py @@ -1,144 +1,144 @@ -import datetime -import time -import pandas as pd -import requests -from urllib3.exceptions import InsecureRequestWarning -import jq -import json - -# Function to authenticate and get token -def authenticate(device_ip, payload): - auth_url = f"https://{device_ip}:8443/api/v1/auth" - response = requests.post(auth_url, json=payload, verify=False) - if response.status_code == 200: - token = response.json()["data"]["token"] - #print(f"Authentication successful. Token received: {token}") - print(" authenticated!") - return token - else: - print(f"Authentication failed. Status code: {response.status_code}") - return None - -# Function to send PATCH request -def send_patch_request(device_ip, token, connection_string): - headers = { - "mx-api-token": token - } - payload = { - "provisioning": { - "source": "manual", - "connectionString": connection_string, - "enable": True - } - } - patch_url = f"https://{device_ip}:8443/api/v1/azure-iotedge" - response = requests.patch(patch_url, json=payload, headers=headers, verify=False) - if response.status_code == 200: - print(f"PATCH request successful for device {device_ip}") - else: - print(f"Failed to send PATCH request to device {device_ip}. Status code: {response.status_code}") - -# Function to send UPGRADE request -def send_upgrade_request(device_ip, token, upgrade_url): - headers = { - "mx-api-token": token - } - payload = { - "deleteFileAfterInstallComplete": True, - "install": True, - "url": upgrade_url - } - patch_url = f"https://{device_ip}:8443/api/v1/upgrades" - response = requests.post(patch_url, json=payload, headers=headers, verify=False) - if response.status_code == 200: - print(f"POST request successful for device {device_ip}") - else: - print(f"Failed to send POST request to device {device_ip}. Status code: {response.status_code}") - print(response.content) - -# Function to send UPGRADE request -def get_upgrade_jobs(device_ip, token): - headers = { - "mx-api-token": token - } - payload = { - } - patch_url = f"https://{device_ip}:8443/api/v1/upgrades/7" - response = requests.get(patch_url, json=payload, headers=headers, verify=False) - if response.status_code == 200: - #print(f"GET request successful for device {device_ip}") - json_data = json.loads(response.content.decode()) - json_str = json.dumps(json_data, indent=4) - #print(json_str) - print(jq.compile('.data.tasks[0].progress').input(json.loads(json_str)).first(), end="% speed: ") - print(jq.compile('.data.tasks[0].speed').input(json.loads(json_str)).first()) - - else: - print(f"Failed to send GET request to device {device_ip}. Status code: {response.status_code}") - print(response.content) - -# Function to send UPGRADE request -def get_API_1(device_ip, token): - headers = { - "mx-api-token": token - } - payload = { - } - patch_url = f"https://{device_ip}:8443/api/v1/azure-iotedge/messages" - response = requests.get(patch_url, json=payload, headers=headers, verify=False) - if response.status_code == 200: - #print(f"GET request successful for device {device_ip}") - #json_data = json.loads(response.content.decode()) - #json_str = json.dumps(json_data) - #print(jq.compile('.data.completedAt').input(json.loads(json_str)).first()) - parsed = json.loads(response.content.decode()) - print(json.dumps(parsed, indent=4)) - else: - print(f"Failed to send GET request to device {device_ip}. Status code: {response.status_code}") - print(response.content.decode()) - - -# Read the Excel file -# df = pd.read_excel("") -# df = df[df["device_name"].notnull()] - -# Iterate over each row in the DataFrame - -requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning) - -payload_auth = { - "acceptEULA": True, - "name": "", - "password": "" -} -if payload_auth["name"] == "" or payload_auth["password"] == "": - print("Provide the credentials before running the script!") - exit(10) -ip_address = "10.197.35.116" -token = authenticate(ip_address, payload_auth) -upgrade_url = ""#"https://files.thingsprocloud.com/package/moxa-aig-301-series-includes-security-patch-firmware-v1.0.deb.yaml" -if upgrade_url == "": - print("Provide upgrade URL before running the script!") - exit(12) -if token: - while True: - print(datetime.datetime.now().strftime("%H:%M:%S"), end=" ") - get_upgrade_jobs(ip_address, token) - time.sleep(60) -# for index, row in df.iterrows(): -# device_name = row['device_name'] -# device_ip_address_https = row['device_ip_address_http'] -# connection_string = row['connection_string'] -# upgrade_url = "https://files.thingsprocloud.com/package/moxa-aig-301-series-includes-security-patch-firmware-v1.0.deb.yaml" - -# # Authenticate and get token -# payload_auth = { -# "acceptEULA": True, -# "name": "", -# "password": "" -# } -# print(device_name, end="") -# token = authenticate(device_ip_address_https, payload_auth) -# if token: -# get_API(device_ip_address_https, token) -# print("\n") +import datetime +import time +import pandas as pd +import requests +from urllib3.exceptions import InsecureRequestWarning +import jq +import json + +# Function to authenticate and get token +def authenticate(device_ip, payload): + auth_url = f"https://{device_ip}:8443/api/v1/auth" + response = requests.post(auth_url, json=payload, verify=False) + if response.status_code == 200: + token = response.json()["data"]["token"] + #print(f"Authentication successful. Token received: {token}") + print(" authenticated!") + return token + else: + print(f"Authentication failed. Status code: {response.status_code}") + return None + +# Function to send PATCH request +def send_patch_request(device_ip, token, connection_string): + headers = { + "mx-api-token": token + } + payload = { + "provisioning": { + "source": "manual", + "connectionString": connection_string, + "enable": True + } + } + patch_url = f"https://{device_ip}:8443/api/v1/azure-iotedge" + response = requests.patch(patch_url, json=payload, headers=headers, verify=False) + if response.status_code == 200: + print(f"PATCH request successful for device {device_ip}") + else: + print(f"Failed to send PATCH request to device {device_ip}. Status code: {response.status_code}") + +# Function to send UPGRADE request +def send_upgrade_request(device_ip, token, upgrade_url): + headers = { + "mx-api-token": token + } + payload = { + "deleteFileAfterInstallComplete": True, + "install": True, + "url": upgrade_url + } + patch_url = f"https://{device_ip}:8443/api/v1/upgrades" + response = requests.post(patch_url, json=payload, headers=headers, verify=False) + if response.status_code == 200: + print(f"POST request successful for device {device_ip}") + else: + print(f"Failed to send POST request to device {device_ip}. Status code: {response.status_code}") + print(response.content) + +# Function to send UPGRADE request +def get_upgrade_jobs(device_ip, token): + headers = { + "mx-api-token": token + } + payload = { + } + patch_url = f"https://{device_ip}:8443/api/v1/upgrades/7" + response = requests.get(patch_url, json=payload, headers=headers, verify=False) + if response.status_code == 200: + #print(f"GET request successful for device {device_ip}") + json_data = json.loads(response.content.decode()) + json_str = json.dumps(json_data, indent=4) + #print(json_str) + print(jq.compile('.data.tasks[0].progress').input(json.loads(json_str)).first(), end="% speed: ") + print(jq.compile('.data.tasks[0].speed').input(json.loads(json_str)).first()) + + else: + print(f"Failed to send GET request to device {device_ip}. Status code: {response.status_code}") + print(response.content) + +# Function to send UPGRADE request +def get_API_1(device_ip, token): + headers = { + "mx-api-token": token + } + payload = { + } + patch_url = f"https://{device_ip}:8443/api/v1/azure-iotedge/messages" + response = requests.get(patch_url, json=payload, headers=headers, verify=False) + if response.status_code == 200: + #print(f"GET request successful for device {device_ip}") + #json_data = json.loads(response.content.decode()) + #json_str = json.dumps(json_data) + #print(jq.compile('.data.completedAt').input(json.loads(json_str)).first()) + parsed = json.loads(response.content.decode()) + print(json.dumps(parsed, indent=4)) + else: + print(f"Failed to send GET request to device {device_ip}. Status code: {response.status_code}") + print(response.content.decode()) + + +# Read the Excel file +# df = pd.read_excel("") +# df = df[df["device_name"].notnull()] + +# Iterate over each row in the DataFrame + +requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning) + +payload_auth = { + "acceptEULA": True, + "name": "", + "password": "" +} +if payload_auth["name"] == "" or payload_auth["password"] == "": + print("Provide the credentials before running the script!") + exit(10) +ip_address = "10.197.35.116" +token = authenticate(ip_address, payload_auth) +upgrade_url = ""#"https://files.thingsprocloud.com/package/moxa-aig-301-series-includes-security-patch-firmware-v1.0.deb.yaml" +if upgrade_url == "": + print("Provide upgrade URL before running the script!") + exit(12) +if token: + while True: + print(datetime.datetime.now().strftime("%H:%M:%S"), end=" ") + get_upgrade_jobs(ip_address, token) + time.sleep(60) +# for index, row in df.iterrows(): +# device_name = row['device_name'] +# device_ip_address_https = row['device_ip_address_http'] +# connection_string = row['connection_string'] +# upgrade_url = "https://files.thingsprocloud.com/package/moxa-aig-301-series-includes-security-patch-firmware-v1.0.deb.yaml" + +# # Authenticate and get token +# payload_auth = { +# "acceptEULA": True, +# "name": "", +# "password": "" +# } +# print(device_name, end="") +# token = authenticate(device_ip_address_https, payload_auth) +# if token: +# get_API(device_ip_address_https, token) +# print("\n") diff --git a/Python/danish_batch_api.py b/Python/danish_batch_api.py index dccd0fe..9dc1f63 100644 --- a/Python/danish_batch_api.py +++ b/Python/danish_batch_api.py @@ -1,147 +1,302 @@ -import pandas as pd -import requests -from urllib3.exceptions import InsecureRequestWarning -import jq -import json - -# Function to authenticate and get token -def authenticate(device_ip, payload): - auth_url = f"https://{device_ip}:8443/api/v1/auth" - response = requests.post(auth_url, json=payload, verify=False) - if response.status_code == 200: - token = response.json()["data"]["token"] - #print(f"Authentication successful. Token received: {token}") - print(" authenticated!") - return token - else: - print(f"Authentication failed. Status code: {response.status_code}") - return None - -# Function to send PATCH request -def send_patch_request(device_ip, token, connection_string): - headers = { - "mx-api-token": token - } - payload = { - "provisioning": { - "source": "manual", - "connectionString": connection_string, - "enable": True - } - } - patch_url = f"https://{device_ip}:8443/api/v1/azure-iotedge" - response = requests.patch(patch_url, json=payload, headers=headers, verify=False) - if response.status_code == 200: - print(f"PATCH request successful for device {device_ip}") - else: - print(f"Failed to send PATCH request to device {device_ip}. Status code: {response.status_code}") - -# Function to send UPGRADE request -def send_upgrade_request(device_ip, token, upgrade_url): - headers = { - "mx-api-token": token - } - payload = { - "deleteFileAfterInstallComplete": True, - "install": True, - "url": upgrade_url - } - patch_url = f"https://{device_ip}:8443/api/v1/upgrades" - response = requests.post(patch_url, json=payload, headers=headers, verify=False) - if response.status_code == 200: - print(f"POST request successful for device {device_ip}") - else: - print(f"Failed to send POST request to device {device_ip}. Status code: {response.status_code}") - print(response.content) - -# Function to send UPGRADE request -def get_upgrade_jobs(device_ip, token): - headers = { - "mx-api-token": token - } - payload = { - } - patch_url = f"https://{device_ip}:8443/api/v1/upgrades/2" - response = requests.get(patch_url, json=payload, headers=headers, verify=False) - if response.status_code == 200: - #print(f"GET request successful for device {device_ip}") - json_data = json.loads(response.content.decode()) - json_str = json.dumps(json_data) - print(jq.compile('.data.completedAt').input(json.loads(json_str)).first()) - else: - print(f"Failed to send GET request to device {device_ip}. Status code: {response.status_code}") - print(response.content) - -# Function to send UPGRADE request -def put_API(device_ip, token): - headers = { - "mx-api-token": token - } - payload = { - } - patch_url = f"https://{device_ip}:8443/api/v1/azure-iotedge/reset" - response = requests.put(patch_url, json=payload, headers=headers, verify=False) - if response.status_code == 200: - #print(f"GET request successful for device {device_ip}") - #json_data = json.loads(response.content.decode()) - #json_str = json.dumps(json_data) - #print(jq.compile('.data.completedAt').input(json.loads(json_str)).first()) - print(response.content.decode()) - else: - print(f"Failed to send GET request to device {device_ip}. Status code: {response.status_code}") - print(response.content.decode()) - -def get_API(device_ip, token): - headers = { - "mx-api-token": token - } - payload = { - } - patch_url = f"https://{device_ip}:8443/api/v1/azure-iotedge" - response = requests.get(patch_url, json=payload, headers=headers, verify=False) - if response.status_code == 200: - #print(f"GET request successful for device {device_ip}") - #json_data = json.loads(response.content.decode()) - #json_str = json.dumps(json_data) - #print(jq.compile('.data.completedAt').input(json.loads(json_str)).first()) - print(response.content.decode()) - else: - print(f"Failed to send GET request to device {device_ip}. Status code: {response.status_code}") - print(response.content.decode()) - -# Read the Excel file -excel_file_path = "" -if excel_file_path == "": - print("Provide Excel file path before running the script!") - exit(11) -df = pd.read_excel(excel_file_path) -df = df[df["device_name"].notnull()] - -# Iterate over each row in the DataFrame - -requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning) - -for index, row in df.iterrows(): - device_name = row['device_name'] - device_ip_address_https = row['device_ip_address_http'] - connection_string = row['connection_string'] - upgrade_url = "" #https://files.thingsprocloud.com/package/moxa-aig-301-series-includes-security-patch-firmware-v1.0.deb.yaml - if upgrade_url == "": - print("Provide upgrade URL before running the script!") - exit(12) - - - # Authenticate and get token - payload_auth = { - "acceptEULA": True, - "name": "", - "password": "" - } - if payload_auth["name"] == "" or payload_auth["password"] == "": - print("Provide the credentials before running the script!") - exit(10) - print(device_name, end="") - token = authenticate(device_ip_address_https, payload_auth) - if token: - get_API(device_ip_address_https, token) - print("\n") +import pandas as pd +import requests +from urllib3.exceptions import InsecureRequestWarning +import jq +import json + +# Function to authenticate and get token +def authenticate(device_ip, payload): + auth_url = f"https://{device_ip}:8443/api/v1/auth" + response = requests.post(auth_url, json=payload, verify=False) + if response.status_code == 200: + token = response.json()["data"]["token"] + #print(f"Authentication successful. Token received: {token}") + #print(" authenticated!") + return token + else: + print(f"Authentication failed. Status code: {response.status_code}") + return None + +# Function to send PATCH request +def send_patch_request(device_ip, token, connection_string): + headers = { + "mx-api-token": token + } + payload = { + "provisioning": { + "source": "manual", + "connectionString": connection_string, + "enable": True + } + } + patch_url = f"https://{device_ip}:8443/api/v1/azure-iotedge" + response = requests.patch(patch_url, json=payload, headers=headers, verify=False) + if response.status_code == 200: + print(f"PATCH request successful for device {device_ip}") + else: + print(f"Failed to send PATCH request to device {device_ip}. Status code: {response.status_code}") + + +# Function to send PATCH request +def patch_time(device_ip, token): + headers = { + "mx-api-token": token + } + # payload = { + # "ntp": { + # "source": "timeserver", + # "server": "10.84.171.254", + # "enable": True + # } + # } + payload = { + "timezone": "America/Chicago" + } + patch_url = f"https://{device_ip}:8443/api/v1/device/time" + response = requests.patch(patch_url, json=payload, headers=headers, verify=False) + if response.status_code == 200: + json_data = json.loads(response.content.decode()) + time = json_data['data']['time'] + timezone = json_data['data']['timezone'] + last = json_data['data']['lastUpdateTime'] + server = json_data['data']['ntp']['server'] + enabled = json_data['data']['ntp']['enable'] + print(time + " " + timezone + " " + last + " " + server + " " + str(enabled)) + else: + json_data = json.loads(response.content.decode()) + #print(json.dumps(json_data, indent=2)) + + +# Function to send UPGRADE request +def send_upgrade_request(device_ip, token, upgrade_url): + headers = { + "mx-api-token": token + } + payload = { + "download": True, + "install": True, + "url": upgrade_url, + } + patch_url = f"https://{device_ip}:8443/api/v1/upgrades" + response = requests.post(patch_url, json=payload, headers=headers, verify=False) + json_data = json.loads(response.content.decode()) + id = json_data['data']['id'] + return id + +# Function to send UPGRADE request +def get_upgrade_job(device_ip, token, id): + headers = { + "mx-api-token": token + } + payload = { + } + patch_url = f"https://{device_ip}:8443/api/v1/upgrades/{id}" + response = requests.get(patch_url, json=payload, headers=headers, verify=False) + if response.status_code == 200: + #print(f"GET request successful for device {device_ip}") + #print(json_str) + #print(json_data['data'][json_data['count'] - 1]['parameter']['url'], json_data['data'][json_data['count'] - 1]['state'], json_data['count']) + json_data = json.loads(response.content.decode()) + getid = json_data['data']['id'] + created = json_data['data']['createdAt'] + cur_status = json_data['data']['state'] + current_tasks = json_data['data']['completedTask'] + total_tasks = json_data['data']['totalTask'] + print("JOB #" + str(getid) + " " + str(current_tasks) + "/" + str(total_tasks)) + print("CREATED ON: " + str(created)) + print("CURRENT STATUS: " + str(cur_status)) + print(json.dumps(json_data, indent=2)) + else: + print(f"Failed to send GET request to device {device_ip}. Status code: {response.status_code}") + print(response.content) + +def get_upgrade_jobs(device_ip, token): + headers = { + "mx-api-token": token + } + payload = { + } + patch_url = f"https://{device_ip}:8443/api/v1/upgrades" + response = requests.get(patch_url, json=payload, headers=headers, verify=False) + if response.status_code == 200: + #print(f"GET request successful for device {device_ip}") + #print(json_str) + #print(json_data['data'][json_data['count'] - 1]['parameter']['url'], json_data['data'][json_data['count'] - 1]['state'], json_data['count']) + json_data = json.loads(response.content.decode()) + count = json_data['count'] + print(str(count)) + for i in range(count): + getid = json_data['data'][i]['id'] + created = json_data['data'][i]['createdAt'] + cur_status = json_data['data'][i]['state'] + current_tasks = json_data['data'][i]['completedTask'] + total_tasks = json_data['data'][i]['totalTask'] + print("JOB #" + str(getid) + " " + str(current_tasks) + "/" + str(total_tasks)) + print("CREATED ON: " + str(created)) + print("CURRENT STATUS: " + str(cur_status)) + else: + print(f"Failed to send GET request to device {device_ip}. Status code: {response.status_code}") + print(response.content) + +# Function to send UPGRADE request +def start_upgrade_job(device_ip, token, id): + headers = { + "mx-api-token": token + } + payload = { + } + patch_url = f"https://{device_ip}:8443/api/v1/upgrades/{id}/start" + response = requests.put(patch_url, json=payload, headers=headers, verify=False) + if response.status_code == 200: + json_data = json.loads(response.content.decode()) + curid = json_data['data']['id'] + startedat = json_data['data']['startedAt'] + print("Job #" + str(curid) + " started on " + str(startedat)) + else: + print(f"Failed to send GET request to device {device_ip}. Status code: {response.status_code}") + print(response.content.decode()) + +# Function to send UPGRADE request +def put_API(device_ip, token): + headers = { + "mx-api-token": token + } + payload = { + } + patch_url = f"https://{device_ip}:8443/api/v1/upgrades/3/start" + response = requests.put(patch_url, json=payload, headers=headers, verify=False) + if response.status_code == 200: + #print(f"GET request successful for device {device_ip}") + #json_data = json.loads(response.content.decode()) + #json_str = json.dumps(json_data) + #print(jq.compile('.data.completedAt').input(json.loads(json_str)).first()) + json_data = json.loads(response.content.decode()) + print(json_data['data']['firmwareVersion']) + else: + print(f"Failed to send GET request to device {device_ip}. Status code: {response.status_code}") + print(response.content.decode()) + +def get_API(device_ip, token): + headers = { + "mx-api-token": token + } + payload = { + } + patch_url = f"https://{device_ip}:8443/api/v1/device/general" + response = requests.get(patch_url, json=payload, headers=headers, verify=False) + if response.status_code == 200: + #print(f"GET request successful for device {device_ip}") + #json_data = json.loads(response.content.decode()) + #json_str = json.dumps(json_data) + #print(jq.compile('.data.completedAt').input(json.loads(json_str)).first()) + json_data = json.loads(response.content.decode()) + hostname = json_data['data']['hostName'] + serial = json_data['data']['serialNumber'] + version = json_data['data']['firmwareVersion'] + description = json_data['data']['description'] + print(hostname + " " + serial + " " + version + " " + description) + #json_str = json.dumps(json_data, indent=2) + #print(json_str) + else: + print(f"Failed to send GET request to device {device_ip}. Status code: {response.status_code}") + print(response.content.decode()) + +def get_time(device_ip, token): + headers = { + "mx-api-token": token + } + payload = { + } + patch_url = f"https://{device_ip}:8443/api/v1/device/time" + response = requests.get(patch_url, json=payload, headers=headers, verify=False) + if response.status_code == 200: + #print(f"GET request successful for device {device_ip}") + #json_data = json.loads(response.content.decode()) + #json_str = json.dumps(json_data) + #print(jq.compile('.data.completedAt').input(json.loads(json_str)).first()) + json_data = json.loads(response.content.decode()) + time = json_data['data']['time'] + timezone = json_data['data']['timezone'] + last = json_data['data']['lastUpdateTime'] + print(time + " " + timezone + " " + last) + #json_str = json.dumps(json_data, indent=2) + #print(json_str) + else: + print(f"Failed to send GET request to device {device_ip}. Status code: {response.status_code}") + print(response.content.decode()) + +def delete_API(device_ip, token): + headers = { + "mx-api-token": token + } + payload = { + } + patch_url = f"https://{device_ip}:8443/api/v1/upgrades/5" + response = requests.delete(patch_url, json=payload, headers=headers, verify=False) + print(response.status_code) + json_data = json.loads(response.content.decode()) + json_str = json.dumps(json_data, indent=2) + print(json_str) + +# Read the Excel file +# excel_file_path = "" +# if excel_file_path == "": +# print("Provide Excel file path before running the script!") +# exit(11) +# df = pd.read_excel(excel_file_path) +# df = df[df["device_name"].notnull()] + +# Iterate over each row in the DataFrame + +requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning) + +# for index, row in df.iterrows(): +# device_name = row['device_name'] +# device_ip_address_https = row['device_ip_address_http'] +# connection_string = row['connection_string'] +# upgrade_url = "" #https://files.thingsprocloud.com/package/moxa-aig-301-series-includes-security-patch-firmware-v1.0.deb.yaml +# if upgrade_url == "": +# print("Provide upgrade URL before running the script!") +# exit(12) + + +# # Authenticate and get token +# payload_auth = { +# "acceptEULA": True, +# "name": "", +# "password": "" +# } +# if payload_auth["name"] == "" or payload_auth["password"] == "": +# print("Provide the credentials before running the script!") +# exit(10) +# print(device_name, end="") +# token = authenticate(device_ip_address_https, payload_auth) +# if token: +# get_API(device_ip_address_https, token) +# print("\n") + + +for i in range(193, 222): + upgrade_url = "https://files.thingsprocloud.com/package/Upgrade_AIG-301_2.4.0-4020_IMG_1.4_to_1.5.deb.yaml" + payload_auth = { + "acceptEULA": True, + "name": "admin", + "password": "admin@123" + } + + device_ip_address = str("10.84.171." + str(i)) + + #print(device_ip_address, end="") + token = authenticate(device_ip_address, payload_auth) + if token: + #id = send_upgrade_request(device_ip_address,token,upgrade_url) + #print(id) + #get_upgrade_job(device_ip_address, token, 6) + #start_upgrade_job(device_ip_address, token, id) + #put_API(device_ip_address, token) + #patch_time(device_ip_address,token) + #get_time(device_ip_address, token) + #delete_API(device_ip_address,token) + get_API(device_ip_address, token) + else: + print("Authentication failed!") \ No newline at end of file diff --git a/Python/danish_batch_scp.py b/Python/danish_batch_scp.py index 0e5b905..035a45b 100644 --- a/Python/danish_batch_scp.py +++ b/Python/danish_batch_scp.py @@ -1,104 +1,112 @@ -import pandas as pd -import requests -from urllib3.exceptions import InsecureRequestWarning -import jq -import json -import scp -import paramiko - -def scp_file(local_path, remote_path, hostname, username, password): - try: - # Create a new SSH client - ssh_client = paramiko.SSHClient() - - # Automatically add the server's host key - ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - - # Connect to the server - ssh_client.connect(hostname, username=username, password=password) - - # Use SCP to transfer the file - with scp.SCPClient(ssh_client.get_transport()) as scp_client: - scp_client.put(local_path, remote_path) - - print("File transferred successfully") - - except Exception as e: - print(f"Error: {e}") - - finally: - # Close the SSH connection - ssh_client.close() - -def ssh_execute_command_with_password(hostname, username, password, command): - try: - # Create a new SSH client - ssh_client = paramiko.SSHClient() - - # Automatically add the server's host key - ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - - # Connect to the server - ssh_client.connect(hostname, username=username, password=password) - - # Execute the command with sudo -S to read password from stdin - stdin, stdout, stderr = ssh_client.exec_command('sudo -k -S ' + command) - stdin.write(password + '\n') - stdin.flush() - - # Read the output - output = stdout.read().decode('utf-8') - error = stderr.read().decode('utf-8') - - # Print output and errors, if any - if output: - print("Command output:") - print(output) - if error: - print("Command error:") - print(error) - - print("Command executed successfully") - - except Exception as e: - print(f"Error: {e}") - - finally: - # Close the SSH connection - ssh_client.close() - -# Read the Excel file -excel_file_path = "" -if excel_file_path == "": - print("Provide Excel file path before running the script!") - exit(11) -df = pd.read_excel(excel_file_path) -df = df[df["device_name"].notnull()] - -# Iterate over each row in the DataFrame - -requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning) - -local_file_path = ""#"./azureiotedge_2.4.0-2697_armhf.mpkg" -if local_file_path == "": - print("Provide upgrade file path before running the script!") - exit(12) -remote_file_path = "./." -username = "" -password = "" -if username == "" or password == "": - print("Provide credentials before running the script!") - exit(10) - -command = ""#"appman app install azureiotedge_2.4.0-2697_armhf.mpkg" -if command == "": - print("Provide a command to execute before running the script!") - exit(11) - -for index, row in df.iterrows(): - device_name = row['device_name'] - device_ip_address_https = row['device_ip_address_http'] - print(device_name) - ssh_execute_command_with_password(device_ip_address_https, username, password, command) - print("\n") - +import pandas as pd +import requests +from urllib3.exceptions import InsecureRequestWarning +import json +import scp +import paramiko + +def scp_file(local_path, remote_path, hostname, username, password): + try: + # Create a new SSH client + ssh_client = paramiko.SSHClient() + + # Automatically add the server's host key + ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + # Connect to the server + ssh_client.connect(hostname, username=username, password=password) + + # Use SCP to transfer the file + with scp.SCPClient(ssh_client.get_transport()) as scp_client: + scp_client.put(local_path, remote_path) + + print("File transferred successfully") + + except Exception as e: + print(f"Error: {e}") + + finally: + # Close the SSH connection + ssh_client.close() + +def ssh_execute_command_with_password(hostname, username, password, command): + try: + # Create a new SSH client + ssh_client = paramiko.SSHClient() + + # Automatically add the server's host key + ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + # Connect to the server + ssh_client.connect(hostname, username=username, password=password) + + # Execute the command with sudo -S to read password from stdin + stdin, stdout, stderr = ssh_client.exec_command('sudo -k -S ' + command) + stdin.write(password + '\n') + stdin.flush() + + # Read the output + output = stdout.read().decode('utf-8') + error = stderr.read().decode('utf-8') + + # Print output and errors, if any + if output: + print("Command output:") + print(output) + if error: + print("Command error:") + print(error) + + print("Command executed successfully") + + except Exception as e: + print(f"Error: {e}") + + finally: + # Close the SSH connection + ssh_client.close() + +# Read the Excel file +# excel_file_path = "" +# if excel_file_path == "": +# print("Provide Excel file path before running the script!") +# exit(11) +# df = pd.read_excel(excel_file_path) +# df = df[df["device_name"].notnull()] + +# Iterate over each row in the DataFrame + +requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning) + +local_file_path = "AIG-301_1.5.2-20240625_saft1_armhf.deb" +if local_file_path == "": + print("Provide upgrade file path before running the script!") + exit(12) +remote_file_path = "./." +username = "moxa" +password = "moxa" +if username == "" or password == "": + print("Provide credentials before running the script!") + exit(10) + +command = ""#"appman app install azureiotedge_2.4.0-2697_armhf.mpkg" +if command == "dpkg -i AIG-301_1.5.2-20240625_saft1_armhf.deb": + print("Provide a command to execute before running the script!") + exit(11) + +# for index, row in df.iterrows(): +# device_name = row['device_name'] +# device_ip_address_https = row['device_ip_address_http'] +# print(device_name) +# #ssh_execute_command_with_password(device_ip_address_https, username, password, command) +# print("\n") + + +for i in range(131, 160): + device_ip_address = str("10.84.157." + str(i)) + print(device_ip_address, end="") + if i == 136 or i == 138 or i == 151: + print(" DONE") + else: + print(" TODO") + scp_file(local_file_path, remote_file_path, device_ip_address, username, password) \ No newline at end of file diff --git a/Python/danish_batch_ssh.py b/Python/danish_batch_ssh.py index d899780..4d34dcc 100644 --- a/Python/danish_batch_ssh.py +++ b/Python/danish_batch_ssh.py @@ -1,87 +1,87 @@ -import paramiko -import pandas as pd -import getpass -import re -import time -import json - -def read_excel(filename, column_name): - df = pd.read_excel(filename) - df = df[df["device_name"].notnull()] - global names - global ips - global connections - names = df["device_name"].tolist() - ips = df["device_ip_address_http"].tolist() - connections = df["connection_string"].tolist() - return ips - -def ssh_execute_commands(device_ip, username, password, commands, connection): - ssh_client = paramiko.SSHClient() - ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh_client.connect(device_ip, username=username, password=password) - transport = ssh_client.get_transport() - session = transport.open_session() - session.set_combine_stderr(True) - session.get_pty() - session.exec_command("sudo cat /var/thingspro/data/mx-api-token") - stdin = session.makefile('wb', -1) - stdout = session.makefile('rb', -1) - stdin.write(password + '\n') - stdin.flush() - for line in stdout.read().splitlines(): - if not re.search('[Pp]assword', line.decode()): - token = line.decode() - - # session = transport.open_session() - # session.set_combine_stderr(True) - # session.get_pty() - # session.exec_command('curl -k -X PUT https://127.0.0.1:8443/api/v1/azure-iotedge/reset -H "Content-Type: application/json" -H "mx-api-token: ' + token + '"') - # stdout = session.makefile('rb', -1) - # for line in stdout.read().splitlines(): - # print(line.decode()) - print("\n" + connection + "\n") - - print(token + "\n") - - jq_data = {} - jq_data_2 = {} - jq_data_2["source"] = "manual" - jq_data_2["connectionString"] = connection - jq_data_2["enable"] = True - jq_data["provisioning"] = jq_data_2 - json_object = json.dumps(jq_data) - print(json_object + "\n") - - session = transport.open_session() - session.set_combine_stderr(True) - session.get_pty() - # -H "Content-Type: application/json" -H "mx-api-token:$(token)' - session.exec_command('curl -k -X PATCH https://127.0.0.1:8443/api/v1/azure-iotedge -H "Content-Type: application/json" -H "mx-api-token: ' + token + '" -d "' + str(json_object) + '"') - stdout = session.makefile('rb', -1) - for line in stdout.read().splitlines(): - print(line.decode()) - -def main(): - filename = "" - if filename == "": - print("Provide Excel file path before running the script!") - exit(11) - column_name = "device_ip_address_http" - global names - global ips - global connections - devices = read_excel(filename, column_name) - username = input("Enter SSH username: ") - password = input("Enter SSH password: ") - print(names) - commands = ["sudo bash"] # Add your commands here - - for i, device in enumerate(names): - print(f"Connecting to gateway #{i} {device} ({ips[i]})...") - ssh_execute_commands(ips[i], username, password, commands, connections[i]) - -if __name__ == "__main__": - names = [] - ips = [] - main() +import paramiko +import pandas as pd +import getpass +import re +import time +import json + +def read_excel(filename, column_name): + df = pd.read_excel(filename) + df = df[df["device_name"].notnull()] + global names + global ips + global connections + names = df["device_name"].tolist() + ips = df["device_ip_address_http"].tolist() + connections = df["connection_string"].tolist() + return ips + +def ssh_execute_commands(device_ip, username, password, commands, connection): + ssh_client = paramiko.SSHClient() + ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh_client.connect(device_ip, username=username, password=password) + transport = ssh_client.get_transport() + session = transport.open_session() + session.set_combine_stderr(True) + session.get_pty() + session.exec_command("sudo cat /var/thingspro/data/mx-api-token") + stdin = session.makefile('wb', -1) + stdout = session.makefile('rb', -1) + stdin.write(password + '\n') + stdin.flush() + for line in stdout.read().splitlines(): + if not re.search('[Pp]assword', line.decode()): + token = line.decode() + + # session = transport.open_session() + # session.set_combine_stderr(True) + # session.get_pty() + # session.exec_command('curl -k -X PUT https://127.0.0.1:8443/api/v1/azure-iotedge/reset -H "Content-Type: application/json" -H "mx-api-token: ' + token + '"') + # stdout = session.makefile('rb', -1) + # for line in stdout.read().splitlines(): + # print(line.decode()) + print("\n" + connection + "\n") + + print(token + "\n") + + jq_data = {} + jq_data_2 = {} + jq_data_2["source"] = "manual" + jq_data_2["connectionString"] = connection + jq_data_2["enable"] = True + jq_data["provisioning"] = jq_data_2 + json_object = json.dumps(jq_data) + print(json_object + "\n") + + session = transport.open_session() + session.set_combine_stderr(True) + session.get_pty() + # -H "Content-Type: application/json" -H "mx-api-token:$(token)' + session.exec_command('curl -k -X PATCH https://127.0.0.1:8443/api/v1/azure-iotedge -H "Content-Type: application/json" -H "mx-api-token: ' + token + '" -d "' + str(json_object) + '"') + stdout = session.makefile('rb', -1) + for line in stdout.read().splitlines(): + print(line.decode()) + +def main(): + filename = "" + if filename == "": + print("Provide Excel file path before running the script!") + exit(11) + column_name = "device_ip_address_http" + global names + global ips + global connections + devices = read_excel(filename, column_name) + username = input("Enter SSH username: ") + password = input("Enter SSH password: ") + print(names) + commands = ["sudo bash"] # Add your commands here + + for i, device in enumerate(names): + print(f"Connecting to gateway #{i} {device} ({ips[i]})...") + ssh_execute_commands(ips[i], username, password, commands, connections[i]) + +if __name__ == "__main__": + names = [] + ips = [] + main() diff --git a/Python/server_updating.py b/Python/server_updating.py index 314d5aa..ec1a4fe 100644 --- a/Python/server_updating.py +++ b/Python/server_updating.py @@ -1,83 +1,83 @@ -#!/usr/bin/env python3 -import asyncio -import logging - -import server_async -import time -import math - -from pymodbus.payload import BinaryPayloadBuilder -from pymodbus.constants import Endian - -from pymodbus.datastore import ( - ModbusSequentialDataBlock, - ModbusServerContext, - ModbusSlaveContext, -) - - -_logger = logging.getLogger(__name__) - - -async def updating_task(context): - # parameters - fc_as_hex = 3 - slave_id = 0x00 - address = 0x00 - count = 100 - - # initialization: set values to zero - values = context[slave_id].getValues(fc_as_hex, address, count=count) - values = [0 for v in values] - - # infinite loop updating every register to its sine value (period 1 hour, amplitude according to register number) - while True: - await asyncio.sleep(1) - current_time = time.time() - sine_value = sin_wave(current_time) - # BinaryPayloadBuilder is used to convert a float to two Modbus registers - builder = BinaryPayloadBuilder(byteorder=Endian.LITTLE) - for i in range(address, count): - builder.add_32bit_float((i+1) * sine_value) - payload = builder.to_registers() - # once the registers of all the sine waves are created, update the values in the simulator - context[slave_id].setValues(fc_as_hex, address, payload) - -# sin wave value calculator -period = 3600 -def sin_wave(time_elapsed): - return math.sin(2 * math.pi * time_elapsed / period) - - -# server start setup function -def setup_updating_server(cmdline=None): - # define the registers - num_floats = 100 - float_values = [float(i + 0.5) for i in range(num_floats)] - builder = BinaryPayloadBuilder(byteorder=Endian.LITTLE) - for value in float_values: - builder.add_32bit_float(value) - payload = builder.to_registers() - - # create a Modbus simulator with the previously generated registers - datablock = ModbusSequentialDataBlock(0x01, payload) - context = ModbusSlaveContext(di=datablock, co=datablock, hr=datablock, ir=datablock) - context = ModbusServerContext(slaves=context, single=True) - return server_async.setup_server( - description="Run asynchronous server.", context=context, cmdline=cmdline - ) - -# asynchronous function that will call the values updating function with the arguments (modbus registers layout) of the created server -async def run_updating_server(args): - task = asyncio.create_task(updating_task(args.context)) - await server_async.run_async_server(args) # start the server - task.cancel() - -# main function that will setup the server, start it and run the function what will update the values of the registers -async def main(cmdline=None): - run_args = setup_updating_server(cmdline=cmdline) - await run_updating_server(run_args) - - -if __name__ == "__main__": - asyncio.run(main(), debug=True) +#!/usr/bin/env python3 +import asyncio +import logging + +import server_async +import time +import math + +from pymodbus.payload import BinaryPayloadBuilder +from pymodbus.constants import Endian + +from pymodbus.datastore import ( + ModbusSequentialDataBlock, + ModbusServerContext, + ModbusSlaveContext, +) + + +_logger = logging.getLogger(__name__) + + +async def updating_task(context): + # parameters + fc_as_hex = 3 + slave_id = 0x00 + address = 0x00 + count = 100 + + # initialization: set values to zero + values = context[slave_id].getValues(fc_as_hex, address, count=count) + values = [0 for v in values] + + # infinite loop updating every register to its sine value (period 1 hour, amplitude according to register number) + while True: + await asyncio.sleep(1) + current_time = time.time() + sine_value = sin_wave(current_time) + # BinaryPayloadBuilder is used to convert a float to two Modbus registers + builder = BinaryPayloadBuilder(byteorder=Endian.LITTLE) + for i in range(address, count): + builder.add_32bit_float((i+1) * sine_value) + payload = builder.to_registers() + # once the registers of all the sine waves are created, update the values in the simulator + context[slave_id].setValues(fc_as_hex, address, payload) + +# sin wave value calculator +period = 3600 +def sin_wave(time_elapsed): + return math.sin(2 * math.pi * time_elapsed / period) + + +# server start setup function +def setup_updating_server(cmdline=None): + # define the registers + num_floats = 100 + float_values = [float(i + 0.5) for i in range(num_floats)] + builder = BinaryPayloadBuilder(byteorder=Endian.LITTLE) + for value in float_values: + builder.add_32bit_float(value) + payload = builder.to_registers() + + # create a Modbus simulator with the previously generated registers + datablock = ModbusSequentialDataBlock(0x01, payload) + context = ModbusSlaveContext(di=datablock, co=datablock, hr=datablock, ir=datablock) + context = ModbusServerContext(slaves=context, single=True) + return server_async.setup_server( + description="Run asynchronous server.", context=context, cmdline=cmdline + ) + +# asynchronous function that will call the values updating function with the arguments (modbus registers layout) of the created server +async def run_updating_server(args): + task = asyncio.create_task(updating_task(args.context)) + await server_async.run_async_server(args) # start the server + task.cancel() + +# main function that will setup the server, start it and run the function what will update the values of the registers +async def main(cmdline=None): + run_args = setup_updating_server(cmdline=cmdline) + await run_updating_server(run_args) + + +if __name__ == "__main__": + asyncio.run(main(), debug=True) diff --git a/Python/stat_json_buffer.py b/Python/stat_json_buffer.py index aa65726..d74ee00 100644 --- a/Python/stat_json_buffer.py +++ b/Python/stat_json_buffer.py @@ -1,77 +1,77 @@ -import os -import json -import pandas as pd -import pprint -import matplotlib.pyplot as plt -from datetime import datetime -import numpy as np - -def process_folder(folder_path, table): - """ - Recursively process each file in the folder and its subfolders. - """ - for root, dirs, files in os.walk(folder_path): - for file in files: - file_path = os.path.join(root, file) - process_file(file_path, table) - -def process_file(file_path, table): - """ - Read each line of JSON data from the file and store it in the table. - """ - with open(file_path, 'r') as f: - for line in f: - try: - json_data = json.loads(line) - table.append(json_data) - except json.JSONDecodeError: - print(f"Error decoding JSON in file: {file_path}") - -def main(folder_path): - table = [] - process_folder(folder_path, table) - - # Convert table to pandas DataFrame for easy manipulation - df = pd.DataFrame(table) - index_table = [] - enqueued_table = [] - emission_table = [] - - for index, row in df.iterrows(): - print(index, end=' ') - index_table.append(index) - print(row['EnqueuedTimeUtc'], end=' ') - enqueued_table.append(row['EnqueuedTimeUtc']) - body_table = [] - body_table.append(row['Body']) - body_df = pd.DataFrame(body_table) - for body_index, body_row in body_df.iterrows(): - print(body_row['emissionDate']) - emission_table.append(body_row['emissionDate']) - - emission_dates = [datetime.strptime(date, '%Y-%m-%dT%H:%M:%SZ') for date in emission_table] - enqueued_dates = [datetime.strptime(date[:19] + date[-1], '%Y-%m-%dT%H:%M:%SZ') for date in enqueued_table] - - plt.figure(figsize=(10, 6)) - plt.plot(enqueued_dates, index_table, label='Enqueued') - plt.plot(emission_dates, index_table, label='Emission') - - plt.xlabel('Time') - plt.ylabel('Index') - plt.title('Index vs Time') - plt.legend() - plt.grid(True) - - plt.xticks(rotation=45) - - parts = folder_path.split('/')[-4:] - result = '_'.join(parts) - - figurename = "index_" + result + ".png" - - plt.savefig(figurename, bbox_inches='tight') - - -if __name__ == "__main__": - folder_path = '/mnt/c/Users/QWPHR/Downloads/JSON_BUFFER_7' - main(folder_path) +import os +import json +import pandas as pd +import pprint +import matplotlib.pyplot as plt +from datetime import datetime +import numpy as np + +def process_folder(folder_path, table): + """ + Recursively process each file in the folder and its subfolders. + """ + for root, dirs, files in os.walk(folder_path): + for file in files: + file_path = os.path.join(root, file) + process_file(file_path, table) + +def process_file(file_path, table): + """ + Read each line of JSON data from the file and store it in the table. + """ + with open(file_path, 'r') as f: + for line in f: + try: + json_data = json.loads(line) + table.append(json_data) + except json.JSONDecodeError: + print(f"Error decoding JSON in file: {file_path}") + +def main(folder_path): + table = [] + process_folder(folder_path, table) + + # Convert table to pandas DataFrame for easy manipulation + df = pd.DataFrame(table) + index_table = [] + enqueued_table = [] + emission_table = [] + + for index, row in df.iterrows(): + print(index, end=' ') + index_table.append(index) + print(row['EnqueuedTimeUtc'], end=' ') + enqueued_table.append(row['EnqueuedTimeUtc']) + body_table = [] + body_table.append(row['Body']) + body_df = pd.DataFrame(body_table) + for body_index, body_row in body_df.iterrows(): + print(body_row['emissionDate']) + emission_table.append(body_row['emissionDate']) + + emission_dates = [datetime.strptime(date, '%Y-%m-%dT%H:%M:%SZ') for date in emission_table] + enqueued_dates = [datetime.strptime(date[:19] + date[-1], '%Y-%m-%dT%H:%M:%SZ') for date in enqueued_table] + + plt.figure(figsize=(10, 6)) + plt.plot(enqueued_dates, index_table, label='Enqueued') + plt.plot(emission_dates, index_table, label='Emission') + + plt.xlabel('Time') + plt.ylabel('Index') + plt.title('Index vs Time') + plt.legend() + plt.grid(True) + + plt.xticks(rotation=45) + + parts = folder_path.split('/')[-4:] + result = '_'.join(parts) + + figurename = "index_" + result + ".png" + + plt.savefig(figurename, bbox_inches='tight') + + +if __name__ == "__main__": + folder_path = '/mnt/c/Users/QWPHR/Downloads/JSON_BUFFER_7' + main(folder_path) diff --git a/Python/tranform_export_hierarchy.py b/Python/tranform_export_hierarchy.py new file mode 100644 index 0000000..c1e9fa4 --- /dev/null +++ b/Python/tranform_export_hierarchy.py @@ -0,0 +1,28 @@ +import re + +# Function to transform each line +def transform_line(line): + # Use regex to extract parts of the line + match = re.match(r'"([^"]+)","([^"]+)","\[(.*)\]"', line) + if match: + id_part = match.group(1) + date_part = match.group(2) + categories_part = match.group(3) + + # Remove extra quotes and split the categories + categories = categories_part.replace('""', '"').split(',') + + # Swap categories order and join them with a semicolon + transformed_categories = ','.join(categories[::-1]) + + # Return the transformed line + return f'"{id_part}",{transformed_categories}' + else: + return None + +# Open input file and output file +with open('export_hierarchy', 'r') as infile, open('export_hierarchy_transformed', 'w') as outfile: + for line in infile: + transformed_line = transform_line(line.strip()) + if transformed_line: + outfile.write(transformed_line + '\n') diff --git a/README.md b/README.md index 0bf635e..d87978d 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,2 @@ -# ess-moxa-configuration-tools +# ess-moxa-configuration-tools Repository containing tools, scripts, configurations and command used in the deployment of Moxa AIG-301 devices for I-Sight with Azure IoT Hub. \ No newline at end of file