Upgrade to 1.5.2, tooling for VPN API requests

.gitignore

@@ -0,0 +1,2 @@
+I-Sight_Generated_Files*
+DATAMODEL_*

Moulinette/data_path_config.py

@@ -35,7 +35,7 @@ allowed_name_characters.append('.')
 logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', filename='data_config_debug.log', filemode='w', level=logging.DEBUG, datefmt='%Y%m%d%H%M%S')
 
 dir_name = 'I-Sight_Generated_Files'
-input_datamodel = 'DATAMODEL_1.0.6_MCO2.xlsx'
+input_datamodel = 'DATAMODEL_1.0.6_SASK.xlsx'
 shell_script_name = dir_name + '/I-Sight_Configuration_'
 
 if (os.path.isdir(dir_name)):
@@ -189,21 +189,15 @@ def jq_filter(current_device, dsh):
         if filter in jq_filter_set and filter != "bitfield":
             logging.debug("Creating standard telemetry topic " + filter + ".")
             jq_data = {}
-            jq_data["enable"] = True
+            jq_data["enable"] = False
             jq_data["properties"] = [{"key": "deviceType", "value": "AC_GATEWAY"}, {"key": "cdid", "value": current_device}]
             jq_data["outputTopic"] = filter
-            if row['message_polling_interval']:
+            jq_data["sendOutThreshold"] = {"mode": "bySize", "size": int(128000), "time": int(30), "sizeIdleTimer": {"enable": True, "time": int(30)}}
-                jq_data["pollingInterval"] = int(row['message_polling_interval'])
+            jq_data["minPublishInterval"] = int(0)
-            else:
+            jq_data["samplingMode"] = "allValues"
-                jq_data["pollingInterval"] = int(1)
+            jq_data["customSamplingRate"] = False
-            if row['message_time_limit'] and row['message_size_limit']:
+            jq_data["pollingInterval"] = int(0)
-                jq_data["sendOutThreshold"] = {"size": int(row['message_size_limit']), "time": int(row['message_time_limit'])}
+            jq_data["onChange"] = False
-            elif not row['message_time_limit'] and row['message_size_limit']:
-                jq_data["sendOutThreshold"] = {"size": int(row['message_size_limit']), "time": int(30)}
-            elif row['message_time_limit'] and not row['message_size_limit']:
-                jq_data["sendOutThreshold"] = {"size": int(256000), "time": int(row['message_time_limit'])}
-            else:
-                jq_data["sendOutThreshold"] = {"size": int(256000), "time": int(30)}
             final_filter = row["jq_filter"].replace("***device_name***", current_device)
             cmd_list = {}
             for metric in all_metrics:
@@ -245,7 +239,7 @@ def bitfield_jq_filter(current_device):
                 logging.debug("Creating bitfield specific telemetry topic" + metric['metric_name'] + " for " + str(assets_name[metric["remoteDevId"]]) + ".")
                 final_filter = {}
                 json_request = {}
-                json_request["enable"] = True
+                json_request["enable"] = False
                 json_request["properties"] = [{"key": "deviceType", "value": "AC_GATEWAY"}, {"key": "cdid", "value": current_device}]
                 json_request["outputTopic"] = metric["metric_name"]
                 json_request["pollingInterval"] = 60
@@ -336,7 +330,7 @@ def tpfunc_gen():
 
     package = {}
     package["name"] = "demoDataFunc"
-    package["enabled"] = True
+    package["enabled"] = False
     package["trigger"] = {"driven":"timeDriven","timeDriven":{"mode":"boot","intervalSec":2,"cronJob":""}}
     package["expose"] = {"tags": expose_tags}
     package["executable"] = {"language":"python"}

Python/azure_iot_hub_1.py

@@ -12,14 +12,14 @@ payload = '{"method":"GET", "path":"/device/general"}'
 # pip install azure-iot-hub
 
 # Authenticate to your Azure account
-CONNECTION_STRING = "HostName=IotHub-CUBE-PROD.azure-devices.net;SharedAccessKeyName=iothubowner;SharedAccessKey= ..."
+CONNECTION_STRING = "HostName=IotHub-CUBE-PROD.azure-devices.net;SharedAccessKeyName=iothubowner;SharedAccessKey=...="
 if CONNECTION_STRING == "":
     print("Provide a connection string for the Iot Hub before running the script!")
     exit(13)
 
 
 registry_manager = IoTHubRegistryManager.from_connection_string(CONNECTION_STRING)
-query_spec = QuerySpecification(query="SELECT * FROM devices WHERE tags.site != 'QWE' AND tags.number != '0' AND capabilities.iotEdge = true")
+query_spec = QuerySpecification(query="SELECT * FROM devices WHERE tags.site = 'MYRTLE' AND tags.number != '0' AND capabilities.iotEdge = true")
 
 query_result = registry_manager.query_iot_hub(query_spec)
 devices = []
@@ -38,7 +38,11 @@ for i in ordered_devices:
         try:
             direct_method = CloudToDeviceMethod(method_name=method_name, payload=json.loads(payload))
             response = registry_manager.invoke_device_module_method(device_id=i[1], module_id=module_id, direct_method_request=direct_method)
-            print(str(i[2]), str(i[0]), str(i[1]), response.payload['data']['description'],sep=";")
+            #print(response.payload)
+            if str(i[1]) == str(response.payload['data']['hostName']):
+                print(str(i[2]), str(i[0]), str(i[1]), response.payload['data']['description'], response.payload['data']['firmwareVersion'], sep=";")
+            else:
+                print(str(i[2]), str(i[0]), str(i[1]), response.payload['data']['description'], response.payload['data']['hostName'], response.payload['data']['firmwareVersion'], sep=";")
         except:
             print(str(i[2]), str(i[0]), str(i[1]), "UNREACHABLE",sep=";")
     else:

Python/danish_batch_api.py

@@ -11,7 +11,7 @@ def authenticate(device_ip, payload):
     if response.status_code == 200:
         token = response.json()["data"]["token"]
         #print(f"Authentication successful. Token received: {token}")
-        print(" authenticated!")
+        #print(" authenticated!")
         return token
     else:
         print(f"Authentication failed. Status code: {response.status_code}")
@@ -36,42 +36,126 @@ def send_patch_request(device_ip, token, connection_string):
     else:
         print(f"Failed to send PATCH request to device {device_ip}. Status code: {response.status_code}")
 
+
+# Function to send PATCH request
+def patch_time(device_ip, token):
+    headers = {
+        "mx-api-token": token
+    }
+    # payload = {
+    #     "ntp": {
+    #         "source": "timeserver",
+    #         "server": "10.84.171.254",
+    #         "enable": True
+    #     }
+    # }
+    payload = {
+        "timezone": "America/Chicago"
+    }
+    patch_url = f"https://{device_ip}:8443/api/v1/device/time"
+    response = requests.patch(patch_url, json=payload, headers=headers, verify=False)
+    if response.status_code == 200:
+        json_data = json.loads(response.content.decode())
+        time = json_data['data']['time']
+        timezone = json_data['data']['timezone']
+        last = json_data['data']['lastUpdateTime']
+        server = json_data['data']['ntp']['server']
+        enabled = json_data['data']['ntp']['enable']
+        print(time + " " + timezone + " " + last + " " + server + " " + str(enabled))
+    else:
+        json_data = json.loads(response.content.decode())
+        #print(json.dumps(json_data, indent=2))
+
+
 # Function to send UPGRADE request
 def send_upgrade_request(device_ip, token, upgrade_url):
     headers = {
         "mx-api-token": token
     }
     payload = {
-        "deleteFileAfterInstallComplete": True,
+        "download": True,
         "install": True,
-        "url": upgrade_url
+        "url": upgrade_url,
     }
     patch_url = f"https://{device_ip}:8443/api/v1/upgrades"
     response = requests.post(patch_url, json=payload, headers=headers, verify=False)
-    if response.status_code == 200:
+    json_data = json.loads(response.content.decode())
-        print(f"POST request successful for device {device_ip}")
+    id = json_data['data']['id']
-    else:
+    return id
-        print(f"Failed to send POST request to device {device_ip}. Status code: {response.status_code}")
-        print(response.content)
 
 # Function to send UPGRADE request
+def get_upgrade_job(device_ip, token, id):
+    headers = {
+        "mx-api-token": token
+    }
+    payload = {
+    }
+    patch_url = f"https://{device_ip}:8443/api/v1/upgrades/{id}"
+    response = requests.get(patch_url, json=payload, headers=headers, verify=False)
+    if response.status_code == 200:
+        #print(f"GET request successful for device {device_ip}")
+        #print(json_str)
+        #print(json_data['data'][json_data['count'] - 1]['parameter']['url'], json_data['data'][json_data['count'] - 1]['state'], json_data['count'])
+        json_data = json.loads(response.content.decode())
+        getid = json_data['data']['id']
+        created = json_data['data']['createdAt']
+        cur_status = json_data['data']['state']
+        current_tasks = json_data['data']['completedTask']
+        total_tasks = json_data['data']['totalTask']
+        print("JOB #" + str(getid) + " " + str(current_tasks) + "/" + str(total_tasks))
+        print("CREATED ON: " + str(created))
+        print("CURRENT STATUS: " + str(cur_status))
+        print(json.dumps(json_data, indent=2))
+    else:
+        print(f"Failed to send GET request to device {device_ip}. Status code: {response.status_code}")
+        print(response.content)
+
 def get_upgrade_jobs(device_ip, token):
     headers = {
         "mx-api-token": token
     }
     payload = {
     }
-    patch_url = f"https://{device_ip}:8443/api/v1/upgrades/2"
+    patch_url = f"https://{device_ip}:8443/api/v1/upgrades"
     response = requests.get(patch_url, json=payload, headers=headers, verify=False)
     if response.status_code == 200:
         #print(f"GET request successful for device {device_ip}")
+        #print(json_str)
+        #print(json_data['data'][json_data['count'] - 1]['parameter']['url'], json_data['data'][json_data['count'] - 1]['state'], json_data['count'])
         json_data = json.loads(response.content.decode())
-        json_str = json.dumps(json_data)
+        count = json_data['count']
-        print(jq.compile('.data.completedAt').input(json.loads(json_str)).first())
+        print(str(count))
+        for i in range(count):
+            getid = json_data['data'][i]['id']
+            created = json_data['data'][i]['createdAt']
+            cur_status = json_data['data'][i]['state']
+            current_tasks = json_data['data'][i]['completedTask']
+            total_tasks = json_data['data'][i]['totalTask']
+            print("JOB #" + str(getid) + " " + str(current_tasks) + "/" + str(total_tasks))
+            print("CREATED ON: " + str(created))
+            print("CURRENT STATUS: " + str(cur_status))
     else:
         print(f"Failed to send GET request to device {device_ip}. Status code: {response.status_code}")
         print(response.content)
 
+# Function to send UPGRADE request
+def start_upgrade_job(device_ip, token, id):
+    headers = {
+        "mx-api-token": token
+    }
+    payload = {
+    }
+    patch_url = f"https://{device_ip}:8443/api/v1/upgrades/{id}/start"
+    response = requests.put(patch_url, json=payload, headers=headers, verify=False)
+    if response.status_code == 200:
+        json_data = json.loads(response.content.decode())
+        curid = json_data['data']['id']
+        startedat = json_data['data']['startedAt']
+        print("Job #" + str(curid) + " started on " + str(startedat))
+    else:
+        print(f"Failed to send GET request to device {device_ip}. Status code: {response.status_code}")
+        print(response.content.decode())
+
 # Function to send UPGRADE request
 def put_API(device_ip, token):
     headers = {
@@ -79,14 +163,15 @@ def put_API(device_ip, token):
     }
     payload = {
     }
-    patch_url = f"https://{device_ip}:8443/api/v1/azure-iotedge/reset"
+    patch_url = f"https://{device_ip}:8443/api/v1/upgrades/3/start"
     response = requests.put(patch_url, json=payload, headers=headers, verify=False)
     if response.status_code == 200:
         #print(f"GET request successful for device {device_ip}")
         #json_data = json.loads(response.content.decode())
         #json_str = json.dumps(json_data)
         #print(jq.compile('.data.completedAt').input(json.loads(json_str)).first())
-        print(response.content.decode())
+        json_data = json.loads(response.content.decode())
+        print(json_data['data']['firmwareVersion'])
     else:
         print(f"Failed to send GET request to device {device_ip}. Status code: {response.status_code}")
         print(response.content.decode())
@@ -97,51 +182,121 @@ def get_API(device_ip, token):
     }
     payload = {
     }
-    patch_url = f"https://{device_ip}:8443/api/v1/azure-iotedge"
+    patch_url = f"https://{device_ip}:8443/api/v1/device/general"
     response = requests.get(patch_url, json=payload, headers=headers, verify=False)
     if response.status_code == 200:
         #print(f"GET request successful for device {device_ip}")
         #json_data = json.loads(response.content.decode())
         #json_str = json.dumps(json_data)
         #print(jq.compile('.data.completedAt').input(json.loads(json_str)).first())
-        print(response.content.decode())
+        json_data = json.loads(response.content.decode())
+        hostname = json_data['data']['hostName']
+        serial = json_data['data']['serialNumber']
+        version = json_data['data']['firmwareVersion']
+        description = json_data['data']['description']
+        print(hostname + " " + serial + " " + version + " " + description)
+        #json_str = json.dumps(json_data, indent=2)
+        #print(json_str)
     else:
         print(f"Failed to send GET request to device {device_ip}. Status code: {response.status_code}")
         print(response.content.decode())
 
+def get_time(device_ip, token):
+    headers = {
+        "mx-api-token": token
+    }
+    payload = {
+    }
+    patch_url = f"https://{device_ip}:8443/api/v1/device/time"
+    response = requests.get(patch_url, json=payload, headers=headers, verify=False)
+    if response.status_code == 200:
+        #print(f"GET request successful for device {device_ip}")
+        #json_data = json.loads(response.content.decode())
+        #json_str = json.dumps(json_data)
+        #print(jq.compile('.data.completedAt').input(json.loads(json_str)).first())
+        json_data = json.loads(response.content.decode())
+        time = json_data['data']['time']
+        timezone = json_data['data']['timezone']
+        last = json_data['data']['lastUpdateTime']
+        print(time + " " + timezone + " " + last)
+        #json_str = json.dumps(json_data, indent=2)
+        #print(json_str)
+    else:
+        print(f"Failed to send GET request to device {device_ip}. Status code: {response.status_code}")
+        print(response.content.decode())
+
+def delete_API(device_ip, token):
+    headers = {
+        "mx-api-token": token
+    }
+    payload = {
+    }
+    patch_url = f"https://{device_ip}:8443/api/v1/upgrades/5"
+    response = requests.delete(patch_url, json=payload, headers=headers, verify=False)
+    print(response.status_code)
+    json_data = json.loads(response.content.decode())
+    json_str = json.dumps(json_data, indent=2)
+    print(json_str)
+
 # Read the Excel file
-excel_file_path = ""
+# excel_file_path = ""
-if excel_file_path == "":
+# if excel_file_path == "":
-    print("Provide Excel file path before running the script!")
+#     print("Provide Excel file path before running the script!")
-    exit(11)
+#     exit(11)
-df = pd.read_excel(excel_file_path)
+# df = pd.read_excel(excel_file_path)
-df = df[df["device_name"].notnull()]
+# df = df[df["device_name"].notnull()]
 
 # Iterate over each row in the DataFrame
 
 requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
 
-for index, row in df.iterrows():
+# for index, row in df.iterrows():
-    device_name = row['device_name']
+#     device_name = row['device_name']
-    device_ip_address_https = row['device_ip_address_http']
+#     device_ip_address_https = row['device_ip_address_http']
-    connection_string = row['connection_string']
+#     connection_string = row['connection_string']
-    upgrade_url = "" #https://files.thingsprocloud.com/package/moxa-aig-301-series-includes-security-patch-firmware-v1.0.deb.yaml
+#     upgrade_url = "" #https://files.thingsprocloud.com/package/moxa-aig-301-series-includes-security-patch-firmware-v1.0.deb.yaml
-    if upgrade_url == "":
+#     if upgrade_url == "":
-        print("Provide upgrade URL before running the script!")
+#         print("Provide upgrade URL before running the script!")
-        exit(12)
+#         exit(12)
     
     
-    # Authenticate and get token
+#     # Authenticate and get token
+#     payload_auth = {
+#         "acceptEULA": True,
+#         "name": "",
+#         "password": ""
+#     }
+#     if payload_auth["name"] == "" or payload_auth["password"] == "":
+#         print("Provide the credentials before running the script!")
+#         exit(10)
+#     print(device_name, end="")
+#     token = authenticate(device_ip_address_https, payload_auth)
+#     if token:
+#         get_API(device_ip_address_https, token)
+#         print("\n")
+
+
+for i in range(193, 222):
+    upgrade_url = "https://files.thingsprocloud.com/package/Upgrade_AIG-301_2.4.0-4020_IMG_1.4_to_1.5.deb.yaml"
     payload_auth = {
         "acceptEULA": True,
-        "name": "",
+        "name": "admin",
-        "password": ""
+        "password": "admin@123"
     }
-    if payload_auth["name"] == "" or payload_auth["password"] == "":
+
-        print("Provide the credentials before running the script!")
+    device_ip_address = str("10.84.171." + str(i))
-        exit(10)
+
-    print(device_name, end="")
+    #print(device_ip_address, end="")
-    token = authenticate(device_ip_address_https, payload_auth)
+    token = authenticate(device_ip_address, payload_auth)
     if token:
-        get_API(device_ip_address_https, token)
+        #id = send_upgrade_request(device_ip_address,token,upgrade_url)
-        print("\n")
+        #print(id)
+        #get_upgrade_job(device_ip_address, token, 6)
+        #start_upgrade_job(device_ip_address, token, id)
+        #put_API(device_ip_address, token)
+        #patch_time(device_ip_address,token)
+        #get_time(device_ip_address, token)
+        #delete_API(device_ip_address,token)
+        get_API(device_ip_address, token)
+    else:
+        print("Authentication failed!")

Python/danish_batch_scp.py

@@ -1,7 +1,6 @@
 import pandas as pd
 import requests
 from urllib3.exceptions import InsecureRequestWarning
-import jq
 import json
 import scp
 import paramiko
@@ -68,37 +67,46 @@ def ssh_execute_command_with_password(hostname, username, password, command):
         ssh_client.close()
 
 # Read the Excel file
-excel_file_path = ""
+# excel_file_path = ""
-if excel_file_path == "":
+# if excel_file_path == "":
-    print("Provide Excel file path before running the script!")
+#     print("Provide Excel file path before running the script!")
-    exit(11)
+#     exit(11)
-df = pd.read_excel(excel_file_path)
+# df = pd.read_excel(excel_file_path)
-df = df[df["device_name"].notnull()]
+# df = df[df["device_name"].notnull()]
 
 # Iterate over each row in the DataFrame
 
 requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
 
-local_file_path = ""#"./azureiotedge_2.4.0-2697_armhf.mpkg"
+local_file_path = "AIG-301_1.5.2-20240625_saft1_armhf.deb"
 if local_file_path == "":
     print("Provide upgrade file path before running the script!")
     exit(12)
 remote_file_path = "./."
-username = ""
+username = "moxa"
-password = ""
+password = "moxa"
 if username == "" or password == "":
     print("Provide credentials before running the script!")
     exit(10)
 
 command = ""#"appman app install azureiotedge_2.4.0-2697_armhf.mpkg"
-if command == "":
+if command == "dpkg -i AIG-301_1.5.2-20240625_saft1_armhf.deb":
     print("Provide a command to execute before running the script!")
     exit(11)
 
-for index, row in df.iterrows():
+# for index, row in df.iterrows():
-    device_name = row['device_name']
+#     device_name = row['device_name']
-    device_ip_address_https = row['device_ip_address_http']
+#     device_ip_address_https = row['device_ip_address_http']
-    print(device_name)
+#     print(device_name)
-    ssh_execute_command_with_password(device_ip_address_https, username, password, command)
+#     #ssh_execute_command_with_password(device_ip_address_https, username, password, command)
-    print("\n")
+#     print("\n")
 
+
+for i in range(131, 160):
+    device_ip_address = str("10.84.157." + str(i))
+    print(device_ip_address, end="")
+    if i == 136 or i == 138 or i == 151:
+        print(" DONE")
+    else:
+        print(" TODO")
+        scp_file(local_file_path, remote_file_path, device_ip_address, username, password)

Python/tranform_export_hierarchy.py

@@ -0,0 +1,28 @@
+import re
+
+# Function to transform each line
+def transform_line(line):
+    # Use regex to extract parts of the line
+    match = re.match(r'"([^"]+)","([^"]+)","\[(.*)\]"', line)
+    if match:
+        id_part = match.group(1)
+        date_part = match.group(2)
+        categories_part = match.group(3)
+
+        # Remove extra quotes and split the categories
+        categories = categories_part.replace('""', '"').split(',')
+        
+        # Swap categories order and join them with a semicolon
+        transformed_categories = ','.join(categories[::-1])
+
+        # Return the transformed line
+        return f'"{id_part}",{transformed_categories}'
+    else:
+        return None
+
+# Open input file and output file
+with open('export_hierarchy', 'r') as infile, open('export_hierarchy_transformed', 'w') as outfile:
+    for line in infile:
+        transformed_line = transform_line(line.strip())
+        if transformed_line:
+            outfile.write(transformed_line + '\n')