General update on Carling tools, device creation on IoT Hub and I-Sight Configuration Tool

.gitignore

@@ -11,4 +11,9 @@ lib
 lib64
 pyvenv.cfg
 *.pem
-*.csv
+*.csv
+/build/*
+Python/build/*
+Python/dist/*
+*.spec
+dist/*

Moulinette/data_path_config.py

@@ -470,7 +470,8 @@ def main():
                     exists = True
                     existing = a + 1
                     logging.debug("Asset " + row_asset['asset_name'] + " not created because it already exists. Processing it under the existing asset.")
-                    excel_parser(row_asset['asset_type'], existing, row_asset["asset_name"], allowed_name_characters, dsh, dsh_global)
+                    print(f"Excel parsing for {row_asset['asset_type']}")
+                    excel_parser(row_asset['asset_type'], existing, row_asset["asset_name"], allowed_name_characters, dsh, dsh_global, row_device)
                     break
 
             if exists != True:

Python/carling_cube.py

@@ -0,0 +1,190 @@
+import csv
+import paramiko
+import time
+from cube_activate_ssh import activate_ssh
+from dotenv import load_dotenv
+import os
+import re
+from azure.iot.hub import IoTHubRegistryManager
+from azure.iot.hub.models import Twin, TwinProperties
+
+load_dotenv(override=True)
+
+ip_address_prefix = "10.81.35."
+
+csv_filename = "carling.csv"
+SITE_NAME = "CARLING"
+
+ssh_username = os.getenv("DEFAULT_CUBE_LINUX_ADMIN_USER")
+ssh_password = os.getenv("DEFAULT_CUBE_LINUX_ADMIN_PASSWORD")
+CONNECTION_STRING = str(os.getenv("CONNECTION_STRING_INOX_PROD"))
+
+def execute_ssh_command(ip, command):
+    client = paramiko.SSHClient()
+    client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    try:
+        client.connect(ip, port=11022, username=ssh_username, password=ssh_password, allow_agent=False, look_for_keys=False)
+        stdin, stdout, stderr = client.exec_command(command)
+        result = stdout.read().decode().lower().strip()
+        return result
+    except Exception as e:
+        print(f"SSH Error: {str(e)}")
+        raise
+    finally:
+        client.close()
+
+def update_cloud_config(ip, new_content):
+    client = paramiko.SSHClient()
+    client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    try:
+        client.connect(ip, port=11022, username=ssh_username, password=ssh_password, allow_agent=False, look_for_keys=False)
+        stdin, stdout, stderr = client.exec_command(f'sudo -S bash -c \'cat > /etc/cube/config-azure.properties << EOF\n{new_content}\nEOF\'\n')
+        stdin.write(ssh_password + "\n")
+        stdin.flush()
+        stdoutput = [line for line in stdout]
+        stderroutput = [line for line in stderr]
+        for output in stdoutput:
+            print(output.strip())
+    except Exception as e:
+        print(f"SSH Error: {str(e)}")
+        raise
+    finally:
+        client.close()
+
+def restart_cloudagent(ip):
+    client = paramiko.SSHClient()
+    client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    try:
+        client.connect(ip, port=11022, username=ssh_username, password=ssh_password, allow_agent=False, look_for_keys=False)
+        stdin, stdout, stderr = client.exec_command(f'sudo -S bash -c \'systemctl restart cube-web-cloudagent << EOF\n\nEOF\'\n')
+        stdin.write(ssh_password + "\n")
+        stdin.flush()
+        stdoutput = [line for line in stdout]
+        stderroutput = [line for line in stderr]
+        for output in stdoutput:
+            print(output.strip())
+    except Exception as e:
+        print(f"SSH Error: {str(e)}")
+        raise
+    finally:
+        client.close()
+
+def main():
+
+    print("Starting...")
+
+    with open(csv_filename, mode="w", newline="") as file:
+        writer = csv.writer(file)
+        writer.writerow(["Number", "IP address", "Cube ID", "Environment", "Correct configuration"])
+
+        registry_manager = IoTHubRegistryManager.from_connection_string(CONNECTION_STRING)
+
+        results = []
+
+        numbers = range(65, 76)
+        for i in numbers:
+            ip_address = f"{ip_address_prefix}{i}"
+            print(f"Activating SSH for {ip_address}:", end=" ")
+            
+            try:
+                activate_ssh(ip_address)
+            except Exception as e:
+                print("Failed!")
+                writer.writerow([i, ip_address, "UNREACHABLE (HTTPS)", "NA", "NA"])
+                file.flush()
+                continue
+
+            print("Activated!")
+
+            ssh_command = "hostname"
+            print(f"Executing {ssh_command} for {ip_address}:", end=" ")
+            try:
+                cube_id = execute_ssh_command(ip_address, ssh_command)
+            except Exception as e:
+                print("Failed!")
+                writer.writerow([i, ip_address, "UNREACHABLE (SSH)", "NA", "NA"])
+                file.flush()
+                continue
+            print(cube_id)
+
+            print(f"Getting configured Connection String")
+            try:
+                connection_string = execute_ssh_command(ip_address, "grep \"connection-string\" /etc/cube/config-azure.properties")
+                if connection_string == "":
+                    raise Exception("No Connection String extracted!")
+
+                iothub_match = re.search(r"hostname\\=(.*?);", connection_string, re.IGNORECASE)
+                iothub = iothub_match.group(1) if iothub_match else None
+                if iothub.lower() == "IotHub-CUBE-PROD.azure-devices.net".lower():
+                    migration = "SAFT"
+                elif iothub.lower() == "iot-ingest-ess-prod.azure-devices.net".lower():
+                    migration = "INOX"
+                else:
+                    migration = "NONE"
+
+                device_id_match =  re.search(r"deviceid\\=(.*?);", connection_string, re.IGNORECASE)
+                cloud_cube_id = device_id_match.group(1) if device_id_match else None
+                if cloud_cube_id.lower() == cube_id.lower():
+                    status = "CORRECT"
+                else:
+                    status = "INCORRECT"
+
+            except Exception as e:
+                print(e)
+                migration = "NONE"
+                status = "INCORRECT"
+
+            if migration == "SAFT" or migration == "NONE":
+                print("SAFT device, migrating to INOX...")
+
+                print("Creating device on INOX...")
+                try:
+                    registry_manager.create_device_with_sas(
+                        cube_id,
+                        primary_key="", secondary_key="",
+                        status="enabled",
+                        iot_edge=False
+                    )
+                except Exception as iot_e:
+                    print("Error creating new device!")
+                    print(iot_e)
+                    continue
+
+                print("Adding tags to new device...")
+                try:
+                    twin = registry_manager.get_twin(cube_id)
+                    twin_patch = Twin(properties=TwinProperties(desired={}), tags={
+                        "site": SITE_NAME,
+                        "number": i
+                    })
+                    registry_manager.update_twin(cube_id, twin_patch, twin.etag)
+                except Exception as iot_e:
+                    print("Error assigning tags to new device!")
+                    print(iot_e)
+
+                print("Requesting primary key...")
+                try:
+                    device_info = registry_manager.get_device(cube_id)
+                    primary_key = device_info.authentication.symmetric_key.primary_key
+                    new_connection_string = f"HostName\\={CONNECTION_STRING.split(';')[0].split('=')[1]};DeviceId\\={cube_id};SharedAccessKey\\={primary_key}"
+                    new_content = f'light-telemetry=false\ncompression-enabled=true\ntelemetry-on=true\nremote-update-on=true\nconnection-string={new_connection_string}'
+                except Exception as iot_e:
+                    print("Error getting new Connection String!")
+                    print(iot_e)
+                    continue
+                print("Setting new Connection String...")
+                try:
+                    update_cloud_config(ip_address, new_content)
+                except Exception as ssh_e:
+                    print("Error when setting the new Connection String!")
+                    print(ssh_e)
+                    continue
+            print("Done!")
+
+            restart_cloudagent(ip_address)
+
+            writer.writerow([i, ip_address, cube_id, migration, status])
+            file.flush()
+
+if __name__ == "__main__":
+    main()

Python/create_iot_hub_devices_from_excel.py

@@ -10,8 +10,8 @@ load_dotenv()
 
 # --- Configuration ---
 IOTHUB_CONNECTION_STRING = os.getenv("CONNECTION_STRING_INOX_PROD")
-INPUT_EXCEL_FILE = 'BELL_DEVICE_LIST.xlsx'
+INPUT_EXCEL_FILE = 'SALINAS_DEVICE_LIST.xlsx'
-OUTPUT_EXCEL_FILE = 'BELL_DEVICE_LIST_CS.xlsx'
+OUTPUT_EXCEL_FILE = 'SALINAS_DEVICE_LIST_CS.xlsx'
 
 def get_connection_string(device: Device, host_name: str) -> str:
     """

Python/cube_activate_ssh.py

@@ -1,14 +1,23 @@
 import requests
 import json
-import argparse
-import sys
 import os
 import urllib3
 from dotenv import load_dotenv
-import io
 import time
+import sys
 
-load_dotenv(override=True)
+def resource_path(relative_path):
+    """ Get absolute path to resource, works for dev and for PyInstaller """
+    try:
+        # PyInstaller creates a temp folder and stores path in _MEIPASS
+        base_path = sys._MEIPASS
+    except Exception:
+        base_path = os.path.abspath(".")
+
+    return os.path.join(base_path, relative_path)
+
+dotenv_path = resource_path('.env')
+load_dotenv(dotenv_path=dotenv_path)
 
 def authenticate(base_url):
     """
@@ -17,9 +26,17 @@ def authenticate(base_url):
     """
     auth_url = f"{base_url}/api/auth"
 
-    username = os.getenv("DEFAULT_CUBE_WEB_ADMIN_USER")
+    ENV_WEB = {
-    password = os.getenv("DEFAULT_CUBE_WEB_ADMIN_PASSWORD")
+        "DEFAULT_CUBE_WEB_ADMIN_USER": os.getenv("DEFAULT_CUBE_WEB_ADMIN_USER"),
-    certificate = os.getenv("DEFAULT_CERTIFICATE").encode("utf-8")
+        "DEFAULT_CUBE_WEB_ADMIN_PASSWORD": os.getenv("DEFAULT_CUBE_WEB_ADMIN_PASSWORD"),
+        "DEFAULT_CERTIFICATE": os.getenv("DEFAULT_CERTIFICATE")
+    }
+
+    username = ENV_WEB["DEFAULT_CUBE_WEB_ADMIN_USER"]
+    password = ENV_WEB["DEFAULT_CUBE_WEB_ADMIN_PASSWORD"]
+    certificate = ENV_WEB["DEFAULT_CERTIFICATE"].encode("utf-8")
+
+    #print(f"{username} {password} {certificate}")
 
     # Prepare the multipart form data
     auth_params = {
@@ -30,7 +47,7 @@ def authenticate(base_url):
         "params": (None, json.dumps(auth_params), "application/json"),
         "certificate": ("certificate.pem", certificate, "application/octet-stream")
     }
-    print(f"Authenticating against {auth_url} with {username} {password} {certificate}")
+    print(f"Authenticating against {auth_url}")
     try:
         response = requests.post(auth_url, files=files, verify=False, timeout=10)
         response.raise_for_status()  # Raise exception for 4XX/5XX responses

Python/cube_ssh_batch_restart_service.py

@@ -4,16 +4,33 @@ from cube_activate_ssh import activate_ssh
 from dotenv import load_dotenv
 import os
 import re
+import sys
 
-load_dotenv(override=True)
+def resource_path(relative_path):
+    """ Get absolute path to resource, works for dev and for PyInstaller """
+    try:
+        # PyInstaller creates a temp folder and stores path in _MEIPASS
+        base_path = sys._MEIPASS
+    except Exception:
+        base_path = os.path.abspath(".")
 
-ip_address_prefix = "10.162.48."
+    return os.path.join(base_path, relative_path)
+
+dotenv_path = resource_path('.env')
+load_dotenv(dotenv_path=dotenv_path)
+
+ip_address_prefix = "10.188.10."
 ssh_command = "hostname"
 
-csv_filename = "lacabana01.csv"
+csv_filename = "HOOHANA_01.csv"
 
-ssh_username = os.getenv("DEFAULT_CUBE_LINUX_ADMIN_USER")
+ENV_SSH = {
-ssh_password = os.getenv("DEFAULT_CUBE_LINUX_ADMIN_PASSWORD")
+    "DEFAULT_CUBE_LINUX_ADMIN_USER": os.getenv("DEFAULT_CUBE_LINUX_ADMIN_USER"),
+    "DEFAULT_CUBE_LINUX_ADMIN_PASSWORD": os.getenv("DEFAULT_CUBE_LINUX_ADMIN_PASSWORD")
+}
+
+ssh_username = ENV_SSH["DEFAULT_CUBE_LINUX_ADMIN_USER"]
+ssh_password = ENV_SSH["DEFAULT_CUBE_LINUX_ADMIN_PASSWORD"]
 
 def execute_ssh_command(ip, command):
     client = paramiko.SSHClient()
@@ -68,6 +85,7 @@ def restart_cloudagent(ip):
 def main():
 
     print("Starting...")
+    #print(f"{ssh_username} {ssh_password}")
 
     with open(csv_filename, mode="w", newline="") as file:
         writer = csv.writer(file)
@@ -125,55 +143,6 @@ def main():
                 migration = "NONE"
                 status = "INCORRECT"
 
-            # if migration == "SAFT" or migration == "NONE":
-            #     print("SAFT device, migrating to INOX...")
-
-            #     print("Creating device on INOX...")
-            #     try:
-            #         registry_manager.create_device_with_sas(
-            #             cube_id,
-            #             primary_key="", secondary_key="",
-            #             status="enabled",
-            #             iot_edge=False
-            #         )
-            #     except Exception as iot_e:
-            #         print("Error creating new device!")
-            #         print(iot_e)
-            #         continue
-
-            #     print("Adding tags to new device...")
-            #     try:
-            #         twin = registry_manager.get_twin(cube_id)
-            #         twin_patch = Twin(properties=TwinProperties(desired={}), tags={
-            #             "site": SITE_NAME,
-            #             "number": i
-            #         })
-            #         registry_manager.update_twin(cube_id, twin_patch, twin.etag)
-            #     except Exception as iot_e:
-            #         print("Error assigning tags to new device!")
-            #         print(iot_e)
-
-            #     print("Requesting primary key...")
-            #     try:
-            #         device_info = registry_manager.get_device(cube_id)
-            #         primary_key = device_info.authentication.symmetric_key.primary_key
-            #         new_connection_string = f"HostName\\={CONNECTION_STRING.split(';')[0].split('=')[1]};DeviceId\\={cube_id};SharedAccessKey\\={primary_key}"
-            #         new_content = f'light-telemetry=false\ncompression-enabled=true\ntelemetry-on=true\nremote-update-on=true\nconnection-string={new_connection_string}'
-            #     except Exception as iot_e:
-            #         print("Error getting new Connection String!")
-            #         print(iot_e)
-            #         continue
-            #     print("Setting new Connection String...")
-            #     try:
-            #         update_cloud_config(ip_address, new_content)
-            #     except Exception as ssh_e:
-            #         print("Error when setting the new Connection String!")
-            #         print(ssh_e)
-            #         continue
-            # print("Done!")
-
-            restart_cloudagent(ip_address)
-
             writer.writerow([i, ip_address, cube_id, migration, status])
             file.flush()
 

Python/requirements.txt

@@ -2,4 +2,6 @@ azure-core
 azure-iot-hub
 python-dotenv
 paramiko
-requests
+requests
+pandas
+openpyxl