Development scripts and docs improvements

DEVELOPMENT.md

@@ -213,6 +213,42 @@ Sample ansible.cfg is there to ensure collection does not need to be installed.
 ansible-playbook -i localhost, examples/iso_info.yml -v
 ```
 
+## Running integration tests
+
+Some test objects (ISO image, VirtualDisk, VM) needs to be created before we run integration tests.
+Use command:
+
+```bash
+ansible-playbook -i localhost, tests/integration/prepare/prepare_vm.yml -v
+ansible-playbook -i localhost, tests/integration/prepare/prepare_iso.yml -v
+ansible-playbook -i localhost, tests/integration/prepare/prepare_examples.yml -v
+```
+
+The `ansible-test integration` will try to run all integration test, and will fail on first problematic test.
+It does allow you to continue from the failed test.
+
+You might want to automatically continue running remaining tests.
+A few failed tests can be reviewed and retried later.
+The `./ci-infra/helpers/run-tests.sh` was made for this.
+If N tests fail in first pass, the `run-tests.sh` will retry only those N test in second pass.
+The script is used like:
+
+```bash
+source ci-infra/local-dev/env-host-4.sh
+./ci-infra/helpers/run-tests.sh outdir <tests.txt>
+```
+
+File `tests.txt` is optional input.
+It contains one test name per line.
+If ommited, all tests from `tests/integration` are run.
+
+After run, the scripts create in `outd` directory:
+
+- directory `log-${timestamp}` directory containing a log file for each run test
+- file `status.txt` contains OK/ERR/PEND/SKIP status for each run test
+
+On next run, only tests that have PEND/ERR status are retried.
+
 ## Creating a release
 
 Releases are automatically created when a tag is created with a name matching
@@ -231,7 +267,7 @@ ScaleComputing does setup new VSNS, with suitable HyperCore version installed.
 
 Steps:
  - Request / reserve static IP address from Alex
-   - either replacing existing static IP or using next in series 105.11.20x
+   - either replacing existing static IP or using next in series 10.5.11.20x
  - create empty VM with 1 virtio disk, type other, tag hc3nested, 16GB ram, 4 cores.
  - image new vSNS node using test iso image (vs. release - this may change in upcoming releases)
  - (optional) Save it as template VM, example name `vsns9213-unconfigured`
@@ -240,6 +276,7 @@ Steps:
    - `sudo singleNodeCluster=1 scclusterinit`
  - Save it as template VM, example name `vsns9213-template`
  - Create a final vSNS from template VM, example name `vsns9213-ci`
+   - keep same MAC address?
  - Add vSNS login URL to Azure OIDC redirectUris
    - ensure ip address is added to entraAD (azure) app registration for OIDC integration (ask Dave if needed)
       - "app_display_name": "Scale Computing HC3",

ci-infra/helpers/run-tests.sh

@@ -5,13 +5,16 @@
 # Input:
 # - outdir will contain status/progress file, and log files of individual tests
 # - (optional) test-names.txt contains integration tests to be run, one per line.
+#   If not given, all tests in tests/integration/targets/ are run.
 # Output:
 # - logs are one test per file, in outdir/log-timestamp/
 # - list of succeded/failed tests are in outdir/status.txt
 # If line starts with:
 # - PEND (or contains just test name), test will be run
 # - SKIP means skip this test
 # - OK or ERR are set after test is run
+# On second run, only PEND tests are run.
+# If there are no PEND tests, the ERR tests are retried.
 
 set -ue
 # set -v
@@ -40,14 +43,29 @@ then
     echo "ERROR file content $TSTATUS" 1>&2
     exit 1
 fi
+/bin/cp "$TSTATUS" "$OUTD2/status.txt"
 
 TEST_NAMES=$(grep "^PEND" "$TSTATUS" | awk '{print $2}')
+if [ -z "$TEST_NAMES" ]
+then
+    echo "No PEND tests, retrying ERR tests"
+    sed -i 's/^ERR/PEND/' "$TSTATUS"
+fi
+TEST_NAMES=$(grep "^PEND" "$TSTATUS" | awk '{print $2}')
+if [ -z "$TEST_NAMES" ]
+then
+    echo "No tests to run, exiting."
+    exit 0
+fi
 # shellcheck disable=SC2086
 echo "Pending tests: "$TEST_NAMES
 # shellcheck disable=SC2086
+TEST_COUNT=$(echo $TEST_NAMES | wc -w)
+
+ii=0
 for TN in $TEST_NAMES
 do
-    echo "Running test $TN"
+    echo "Running test $TN ($ii/$TEST_COUNT)"
     (
         echo ansible-test integration --local "$TN"
         echo "======================"
@@ -67,4 +85,5 @@ do
     ) >"$OUTD2/$TN.log" 2>&1
     res=$(grep $'\t'"$TN\$" "$TSTATUS" | awk '{print $1}')
     echo "  result $res $TN"
+    ii=$((ii+1))
 done

tests/integration/integration_config.yml.j2

@@ -35,6 +35,10 @@ sc_config:
       # pool.ntp.org servers frequently return KOD.
       # time.google.com does not complain.
       source: time.google.com
+      # We need 2 NTP servers.
+      # Set them up per ci-infra/ntp-chrony/Readme.md.
+      time_server_a: 10.5.11.5
+      time_server_b: 10.5.11.75
     time_zone:
       zone: US/Eastern
     dns_config:

tests/integration/targets/i_vm_params/tasks/02_machine_type.yml

@@ -210,6 +210,12 @@
         - not output.records[0].snapshot_schedule
 
   # ===================================================================
+  # Reset a running VM
+  - name: Check initial VM power_state
+    ansible.builtin.assert:
+      that:
+        - output.records[0].power_state == "started"
+
   - name: Set VMs power_state - reset
     scale_computing.hypercore.vm_params:
       vm_name: "{{ vm_name }}"

tests/integration/targets/inventory/runme.sh

@@ -55,6 +55,8 @@ ansible-playbook -i localhost, -i hypercore_inventory_ansible_both_false.yml run
 
 # test with OIDC user
 # We can do this only if OIDC login is configured.
+echo "Ensure OIDC login is setup on tested HyperCore."
+ansible-playbook -e @"$vars_file" set_oidc_config.yml
 echo "Testing inventory plugin with OIDC user."
 eval "$(cat <<EOF | python
 import yaml

tests/integration/targets/inventory/set_oidc_config.yml

@@ -0,0 +1,28 @@
+---
+- name: Set OIDC configuration
+  hosts: localhost
+  gather_facts: False
+  vars:
+    oidc_config: "{{ sc_config[sc_host].oidc }}"
+
+  tasks:
+    - name: Set OIDC configuration
+      scale_computing.hypercore.oidc_config:
+        client_id: "{{ oidc_config.client_id_default }}"
+        shared_secret: "{{ oidc_config.shared_secret_default }}"
+        config_url: "{{ oidc_config.config_url_default }}"
+        scopes: "openid+profile"
+
+    - name: Get OIDC configuration
+      scale_computing.hypercore.oidc_config:
+        action: get
+        endpoint: /rest/v1/OIDCConfig
+      register: oidc_config_response
+
+    - name: Assert OIDC is configured
+      ansible.builtin.assert:
+        that:
+          # - oidc_config_response.status == 200
+          - oidc_config_response.record | length == 1
+          - oidc_config_response.record.0.clientID == expected_oidc_client_id
+        fail_msg: "OIDC is not configured on the Scale Computing cluster."

tests/integration/targets/role_template2vm/templates/meta-data.ubuntu-20.04.yml.j2

@@ -1,5 +1,11 @@
 dsmode: local
 local-hostname: "{{ vm_name }}"
+{#
+  Comment out network configuration to use cloud-init default
+  which is to enable DHCP on the first detected interface.
+  In generated /var/lib/cloud/instance/network-config.json is string "_orig_eni_name".
+  This is invalid cloud-init config.
+
 network-interfaces: |
   auto lo
   iface lo inet loopback
@@ -13,6 +19,8 @@ network-interfaces: |
   #   - CI VSNS v9.3.5 (https://10.5.11.203/)
   #   - CI VSNS v9.4.17 (https://10.5.11.205/)
   iface ens1 inet dhcp
+  # Device enp1s1 on VSNS 4.7.0.221603 prerelease
+  iface enp1 inet dhcp
 {% endif %}
 {% if vm_network_mode == "static" %}
   iface {{ vm_network_iface }} inet static
@@ -21,3 +29,4 @@ network-interfaces: |
     gateway {{ vm_network_gateway }}
     dns-nameservers {{ vm_network_dns_nameservers }}
 {% endif %}
+#}

tests/integration/targets/time_server/tasks/01_time_server_info.yml

@@ -6,9 +6,7 @@
 
   vars:
     actual_uuid: "timesource_guid"
-    # We need 2 NTP servers.
-    # Set them up per ci-infra/ntp-chrony/Readme.md.
-    actual_host: "10.5.11.5"
+    actual_host: "{{ sc_config[sc_host].time_server.time_server_a }}"
 
   block:
     # -------------------------------------------------------

tests/integration/targets/time_server/tasks/02_time_server.yml

@@ -5,10 +5,8 @@
     SC_PASSWORD: "{{ sc_config[sc_host].sc_password }}"
     SC_TIMEOUT: "{{ sc_timeout * 10 }}"
   vars:
-    # We need 2 NTP servers.
-    # Set them up per ci-infra/ntp-chrony/Readme.md.
-    time_server_a: 10.5.11.5
-    time_server_b: 10.5.11.75
+    time_server_a: "{{ sc_config[sc_host].time_server.time_server_a }}"
+    time_server_b: "{{ sc_config[sc_host].time_server.time_server_b }}"
     actual_uuid: timesource_guid
 
   block: