fix tests and CI

.github/workflows/test.yml

@@ -1,4 +1,4 @@
-name: Test Suite
+name: MetaCall Examples CI
 
 on:
   workflow_dispatch:
@@ -7,46 +7,79 @@ on:
     tags:
       - 'v*.*.*'
     branches:
-      - master
+      - main
 
 jobs:
-  build:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest, windows-latest] # TODO: macos-latest
-    runs-on: ${{ matrix.os }}
+  LinuxUbuntuRun:
+    name: Linux - Ubuntu Run
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.12'
+
+      - name: Install MetaCall
+        run: |
+          sudo apt update
+          curl -sL https://raw.githubusercontent.com/metacall/install/master/install.sh | sh
 
-      - name: Install MetaCall (Linux)
-        if: matrix.os == 'ubuntu-latest'
-        run: curl -sL https://raw.githubusercontent.com/metacall/install/master/install.sh | sh
+      - name: Install Dependencies
+        run: |
+          pip install -r requirements.txt
+
+      - name: Run Tests Suits
+        run: |
+          pip install -r requirements.txt
+          find test-suites -type f -name "*.yaml" -exec python ./testing.py -f {} -V \;
 
-      - name: Install MetaCall (Windows)
-        if: matrix.os == 'windows-latest'
-        run: powershell -NoProfile -ExecutionPolicy unrestricted -Command "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -UseBasicParsing 'https://raw.githubusercontent.com/metacall/install/master/install.ps1')))"
+  # WindowsRun:
+  #   name: Windows Run
+  #   runs-on: windows-latest
+  #   steps:
+  #     - name: Checkout code
+  #       uses: actions/checkout@v4
 
-      # TODO:
-      # - name: Install MetaCall (MacOS)
-      #   if: matrix.os == 'macos-latest'
-      #   run: curl -sL https://raw.githubusercontent.com/metacall/install/master/install.sh | sh
+  #     - name: Set up Python
+  #       uses: actions/setup-python@v5
+  #       with:
+  #         python-version: '3.12'
+
+  #     - name: Install MetaCall
+  #       shell: pwsh
+  #       run: |
+  #         powershell -NoProfile -ExecutionPolicy unrestricted -Command "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -UseBasicParsing 'https://raw.githubusercontent.com/metacall/install/master/install.ps1')))"
+
+  #     - name: Run Test Suits
+  #       shell: pwsh
+  #       run: |
+  #         pip install pyyaml
+  #         Get-ChildItem -Path test-suits -Filter *.yaml | ForEach-Object { python ./metacall-test.py -f $_.FullName -V }
+  #         Clear-Host
+  #         for /f %f in (failed-test-cases.txt) do type %f && exit /b 1
 
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
+  # MacOSRun:
+  #   name: MacOS Run
+  #   runs-on: macos-latest
+  #   steps:
+  #     - name: Checkout code
+  #       uses: actions/checkout@v4
 
-      - name: Run tests
-        shell: bash
-        run: |
-          for test in ./suites/*.yaml; do
-            echo "Running $test"
-            python3 ./testing.py -f $test -V
-          done
+  #     - name: Set up Python
+  #       uses: actions/setup-python@v5
+  #       with:
+  #         python-version: '3.12'
+
+  #     - name: Install MetaCall
+  #       run: |
+  #         curl -sL https://raw.githubusercontent.com/metacall/install/master/install.sh | sh
+
+  #     - name: Run Tests Suits
+  #       run: |
+  #         pip install pyyaml
+  #         find test-suits -type f -name "*.yaml" -exec python ./metacall-test.py -f {} -V \;
+  #         clear
+  #         for file in failed-test-cases.txt; do [ -s "$file" ] && cat failed-test-cases.txt && exit 1; done

README.md

@@ -12,10 +12,10 @@ code-files:
     test-cases:
       - name: Check the password is generated in the correct length
         command: call getRandomPassword(12)
-        expected-stdout: '\"[\w\W]{12}\"'
+        expected-pattern: '\"[\w\W]{12}\"'
       - name: Check the password is generated in the correct length
         command: call getRandomPassword()
-        expected-stdout: 'missing 1 required positional argument'
+        expected-pattern: 'missing 1 required positional argument'
 ``` 
 
 ## Arguments
@@ -31,5 +31,5 @@ options:
 ```
 Example:
 ```bash
-./testing.py -f  suites/test-time-app-web.yaml -V
+python3 ./testing.py -f  test-suites/test-time-app-web.yaml -V
 ```

TODO.md

@@ -1,11 +1,11 @@
 # ToDos
 
-- [ ] Create a stable version that runs existing tests.
-- [ ] If any test fail, it should exit with error code 1. Right now tests are failing and it is passing with exit code 0.
+- [x] Create a stable version that runs existing tests.
+- [x] If any test fail, it should exit with error code 1. Right now tests are failing and it is passing with exit code 0.
 - [ ] Support FaaS.
 - [ ] Update/fix test cases.
-- [x] Run the pipeline on Linux.
-- [x] Run the pipeline on Windows.
+- [ ] Run the pipeline on Linux.
+- [ ] Run the pipeline on Windows.
 - [ ] Run the pipeline on MacOS.
 - [ ] Link the pipeline with the distributables and install script.
 - [ ] Update the Readme.

suites/test-auth-function-mesh.yaml → test-auth-function-mesh.yaml

@@ -6,10 +6,10 @@ code-files:
     test-cases:
       - name: Check the encrypt function
         command: call encrypt("asd")
-        expected-stdout: 'eyJhbGciOiJIUzI1NiJ9.YXNk.QNa-p8QpuHcVUDMN_Ih4x4vidWp31365GM4zrSr3t0s'
+        expected-pattern: 'eyJhbGciOiJIUzI1NiJ9.YXNk.QNa-p8QpuHcVUDMN_Ih4x4vidWp31365GM4zrSr3t0s'
       - name: Check the decrypt function
         command: call decrypt("eyJhbGciOiJIUzI1NiJ9.YXNk.QNa-p8QpuHcVUDMN_Ih4x4vidWp31365GM4zrSr3t0s")
-        expected-stdout: 'asd'
+        expected-pattern: 'asd'
 
 
 

suites/test-auth-middleware.yaml → test-auth-middleware.yaml

@@ -6,13 +6,13 @@ code-files:
     test-cases:
       - name: Check the sum function
         command: call sum(3,4)
-        expected-stdout: '7'
+        expected-pattern: '7'
       - name: Check the signin function
         command: call signin("Mostafa","1557")
-        expected-stdout: '.*'
+        expected-pattern: '.*'
       - name: Check the reverse function
         command: call reverse("abcdefg")
-        expected-stdout: '.*'
+        expected-pattern: "gfedcba"
 
 
 

suites/test-random-password-generator.yaml → test-suites/test-random-password-generator.yaml

@@ -7,9 +7,9 @@ code-files:
     test-cases:
       - name: Check the password is generated in the correct length
         command: call getRandomPassword(12)
-        expected-stdout: '\"[\w\W]{12}\"'
-      - name: Check the password is generated in the correct length
+        expected-pattern: '[\S]{12}'
+      - name: Check for missing argument
         command: call getRandomPassword()
-        expected-stdout: 'missing 1 required positional argument'
+        expected-pattern: 'missing 1 required positional argument'
 
 

suites/test-string-manipulation.yaml → test-suites/test-string-manipulation.yaml

@@ -6,13 +6,13 @@ code-files:
     test-cases:
       - name: Check the longest_repetition function
         command: call longest_repetition("aaa")
-        expected-stdout: "[ 'a', 3 ]"
+        expected-pattern: "[ 'a', 3 ]"
       - name: Check the longest_repetition function with a different string
         command: call longest_repetition("bbbaaabaaaa")
-        expected-stdout: "[ 'a', 4 ]"
+        expected-pattern: "[ 'a', 4 ]"
       - name: Check the longest_repetition function with an empty string
         command: call longest_repetition("")
-        expected-stdout: '\[null,0\]'
+        expected-pattern: "NodeJS Loader could not convert the value of type 'Invalid' to N-API"
 
 
 

suites/test-time-app-web.yaml → test-suites/test-time-app-web.yaml

@@ -5,13 +5,13 @@ code-files:
     test-cases:
       - name: Check the time is generated in the correct format
         command: call time()
-        expected-stdout: '\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}'
+        expected-pattern: '\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}'
       - name: Check calling the time function with a parameter
         command: call time(1)
-        expected-stdout: 'takes 0 positional arguments but 1 was given'
+        expected-pattern: 'takes 0 positional arguments but 1 was given'
       - name: Check index.html is fully returned
         command: call index()
-        expected-stdout: '<html>[\w\W]*</html>'
+        expected-pattern: '<html>[\w\W]*</html>'
 
 
 

testing.py

@@ -18,13 +18,17 @@ def main():
 
     test_suite_file_name = args.file
 
+
+
     test_suites_extractor = TestSuitesExtractor(test_suite_file_name)
     project_name, repo_url, test_suites = test_suites_extractor.extract_test_suites()
 
+    logger.info(f"Project: {project_name}")
+
     repo_manager = RepoManager(repo_url)
     repo_manager.clone_repo_if_not_exist()
 
-    test_runner = TestRunner("composite")
+    test_runner = TestRunner(["cli"])
     test_runner.run_tests(project_name, test_suites)
 
 if __name__ == "__main__":

testing/logger.py

@@ -15,6 +15,7 @@ def __init__(self):
         else:
             Logger._instance = self
             self.logger = logging.getLogger("CLI_Tool")
+            self.level = "INFO" # default level
             handler = logging.StreamHandler()
             formatter = logging.Formatter('%(levelname)s - %(message)s')
             handler.setFormatter(formatter)
@@ -29,8 +30,12 @@ def set_level(self, level):
             "ERROR": logging.ERROR,
             "CRITICAL": logging.CRITICAL
         }
+        self.level = level.upper()
         self.logger.setLevel(level_map.get(level.upper(), logging.INFO))
 
+    def get_level(self):
+        return self.level
+
     def debug(self, msg):
         self.logger.debug(msg)
 

testing/repo_manager.py

@@ -11,14 +11,15 @@ def clone_repo_if_not_exist(self):
         try:
             repo_name = self.repo_url.split('/')[-1].split('.')[0]
             if os.path.isdir(repo_name):
-                self.logger.debug("Repo is already cloned!")
+                self.logger.warning("Repo is already cloned!")
             else:
                 process = subprocess.Popen(['git', 'clone', self.repo_url], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                 _, stderr = process.communicate()
                 stderr = stderr.decode('utf-8')
-                if "Fatal" in stderr or "fatal" in stderr or "error" in stderr or "Error" in stderr or "ERROR" in stderr:
-                    raise Exception(stderr)
+                error_keywords = ["Fatal", "fatal", "error", "Error", "ERROR"]
+                if any(keyword in stderr for keyword in error_keywords):
+                    raise ValueError(stderr)
                 self.logger.debug("Repo is cloned successfully!")
-        except Exception as e:
+        except ValueError as e:
             self.logger.error(f"Error: {e}")
             exit()

testing/runner/cli_interface.py

@@ -5,6 +5,8 @@
 class CLIInterface(RunnerInterface):
     def __init__(self):
         self.logger = Logger.get_instance()
+    def get_name(self):
+        return "cli"
 
     def get_runtime_tag(self, file_name):
         file_extension = file_name.split('.')[-1]
@@ -21,23 +23,23 @@ def get_runtime_tag(self, file_name):
         else:
             raise ValueError("Error: file extension not supported!")
 
-    def run_test_command(self, filename, file_path, test_case_command):
+    def run_test_command(self, file_path, test_case_command):
+        file_name = file_path.split('/')[-1]
         try:
             process = subprocess.Popen(['metacall'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            self.logger.debug("Metacall CLI started...")
 
-
-            commands = ['load ' + ' ' + self.get_runtime_tag(filename) + ' ' + file_path, test_case_command, 'exit']
+            commands = ['load ' + ' ' + self.get_runtime_tag(file_name) + ' ' + file_path, test_case_command, 'exit']
             commands = '\n'.join(commands) + '\n' # join the commands with a newline character
-
+        
             process.stdin.write(f"{commands}".encode('utf-8'))
             process.stdin.flush()
 
             stdout, _ = process.communicate()
+
             out_str = stdout.decode('utf-8').strip().split('λ')
-            out_str = out_str[2]
-            return out_str
+
+            return out_str[2]
 
-        except Exception as e:
+        except subprocess.CalledProcessError as e:
             self.logger.error(f"Error: {e}")
-            return None
+            return ""

testing/runner/faas_interface.py

@@ -1,7 +1,13 @@
 from testing.runner.runner_interface import RunnerInterface
 
 class FaaSInterface(RunnerInterface):
-    def run_test_command(self, filename, file_path, test_case_command):
+    def __init__(self):
+        pass
+
+    def get_name(self):
+        return "faas"
+
+    def run_test_command(self, file_path, test_case_command):
         # Implement the FaaS call here
         # For now, return a placeholder string
         return "FaaS output placeholder"

testing/runner/interface_factory.py

@@ -1,6 +1,5 @@
 from testing.runner.cli_interface import CLIInterface
 from testing.runner.faas_interface import FaaSInterface
-from testing.runner.composite_runner_interface import CompositeRunnerInterface
 
 class InterfaceFactory:
     @staticmethod
@@ -9,10 +8,5 @@ def get_interface(interface_type):
             return CLIInterface()
         elif interface_type == "faas":
             return FaaSInterface()
-        elif interface_type == "composite":
-            composite = CompositeRunnerInterface()
-            composite.add_runner(CLIInterface())
-            composite.add_runner(FaaSInterface())
-            return composite
         else:
             raise ValueError(f"Unknown interface type: {interface_type}")

testing/runner/runner_interface.py

@@ -2,5 +2,5 @@
 
 class RunnerInterface(ABC):
     @abstractmethod
-    def run_test_command(self, filename, file_path, test_case_command):
+    def run_test_command(self, file_path, test_case_command):
         pass