[DO NOT MERGE][CI ONLY] EBUSY flake investigation

.cirrus.yml

@@ -58,7 +58,7 @@ env:
     DISTRO_NV:               # any {PRIOR_,}{FEDORA,DEBIAN}_NAME value
     VM_IMAGE_NAME:           # One of the "Google-cloud VM Images" (above)
     CTR_FQIN:                # One of the "Container FQIN's" (above)
-    CI_DESIRED_DATABASE: boltdb # One of "", "sqlite", "boltdb"
+    CI_DESIRED_DATABASE: sqlite # One of "", "sqlite", "boltdb"
 
     # Curl-command prefix for downloading task artifacts, simply add the
     # the url-encoded task name, artifact name, and path as a suffix.
@@ -293,6 +293,7 @@ bindings_task:
         $CIRRUS_PR != '' &&
         $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' &&
         $CIRRUS_CHANGE_TITLE !=~ '.*CI:BUILD.*'
+    skip: $CI == $CI
     depends_on:
         - build
     gce_instance: *standardvm
@@ -326,6 +327,7 @@ swagger_task:
     # Don't create task for [CI:BUILD]
     # Docs: ./contrib/cirrus/CIModes.md
     only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:BUILD.*'
+    skip: $CI == $CI
     depends_on:
         - build
     gce_instance: *standardvm
@@ -357,6 +359,7 @@ alt_build_task:
     only_if: &no_rhel_release |
         $CIRRUS_BRANCH !=~ 'v[0-9\.]+-rhel' &&
         $CIRRUS_BASE_BRANCH !=~ 'v[0-9\.]+-rhel'
+    skip: $CI == $CI
     depends_on:
         - build
     env:
@@ -395,6 +398,7 @@ win_installer_task:
       $CIRRUS_TAG == '' &&
       $CIRRUS_BRANCH !=~ 'v[0-9\.]+-rhel' &&
       $CIRRUS_BASE_BRANCH !=~ 'v[0-9\.]+-rhel'
+    skip: $CI == $CI
     depends_on:
       - alt_build
     ec2_instance:
@@ -418,6 +422,7 @@ osx_alt_build_task:
     alias: osx_alt_build
     # Docs: ./contrib/cirrus/CIModes.md
     only_if: *no_rhel_release
+    skip: $CI == $CI
     depends_on:
         - build
     env:
@@ -490,7 +495,7 @@ docker-py_test_task:
         $CIRRUS_PR != '' &&
         $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' &&
         $CIRRUS_CHANGE_TITLE !=~ '.*CI:BUILD.*'
-
+    skip: $CI == $CI
     depends_on:
         - build
     gce_instance: *standardvm
@@ -510,6 +515,7 @@ unit_test_task:
     alias: unit_test
     # Docs: ./contrib/cirrus/CIModes.md
     only_if: *not_tag_branch_build_docs
+    skip: $CI == $CI
     depends_on:
         - build
         - validate
@@ -534,6 +540,7 @@ apiv2_test_task:
     alias: apiv2_test
     # Docs: ./contrib/cirrus/CIModes.md
     only_if: *not_tag_branch_build_docs
+    skip: $CI == $CI
     depends_on:
         - build
         - validate
@@ -560,6 +567,7 @@ compose_test_task:
     alias: compose_test
     # Docs: ./contrib/cirrus/CIModes.md
     only_if: *not_tag_branch_build_docs
+    skip: $CI == $CI
     depends_on:
         - build
         - validate
@@ -596,6 +604,7 @@ windows_smoke_test_task:
         $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' &&
         $CIRRUS_BRANCH !=~ 'v[0-9\.]+-rhel' &&
         $CIRRUS_BASE_BRANCH !=~ 'v[0-9\.]+-rhel'
+    skip: $CI == $CI
     depends_on:
       - alt_build
     ec2_instance:
@@ -711,6 +720,7 @@ podman_machine_task:
         $CIRRUS_TAG == '' &&
         $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' &&
         $CIRRUS_CHANGE_TITLE !=~ '.*CI:BUILD.*'
+    skip: $CI == $CI
     depends_on:
         - build
         - local_integration_test
@@ -738,6 +748,7 @@ podman_machine_aarch64_task:
     name: *std_name_fmt
     alias: podman_machine_aarch64
     only_if: *not_tag_build_docs
+    skip: $CI == $CI
     depends_on:
         - build_aarch64
         - validate_aarch64
@@ -773,6 +784,7 @@ local_system_test_task: &local_system_test_task
     gce_instance: *standardvm
     env:
         TEST_FLAVOR: sys
+    timeout_in: 45m
     clone_script: *get_gosrc
     setup_script: *setup
     main_script: *main
@@ -793,6 +805,7 @@ local_system_test_aarch64_task: &local_system_test_task_aarch64
         <<: *stdenvars_aarch64
         TEST_FLAVOR: sys
         DISTRO_NV: ${FEDORA_AARCH64_NAME}
+    timeout_in: 45m
     clone_script: *get_gosrc_aarch64
     setup_script: *setup
     main_script: *main
@@ -845,6 +858,7 @@ rootless_system_test_task:
     env:
         TEST_FLAVOR: sys
         PRIV_NAME: rootless
+    timeout_in: 45m
     clone_script: *get_gosrc
     setup_script: *setup
     main_script: *main
@@ -855,6 +869,7 @@ minikube_test_task:
     alias: minikube_test
     # Docs: ./contrib/cirrus/CIModes.md
     only_if: *not_tag_build_docs
+    skip: $CI == $CI
     depends_on:
         - build
         - rootless_system_test
@@ -873,6 +888,7 @@ buildah_bud_test_task:
     alias: buildah_bud_test
     # Docs: ./contrib/cirrus/CIModes.md
     only_if: *not_tag_branch_build_docs
+    skip: $CI == $CI
     depends_on:
         - build
         - local_integration_test
@@ -896,6 +912,7 @@ rootless_buildah_bud_test_task:
     alias: rootless_buildah_bud_test
     # Please keep this as-is: the buildah treadmill (#13808) relies on it.
     only_if: $CIRRUS_CRON == 'treadmill'
+    skip: $CI == $CI
     depends_on:
         - build
         - rootless_integration_test
@@ -920,6 +937,7 @@ upgrade_test_task:
     alias: upgrade_test
     # Docs: ./contrib/cirrus/CIModes.md
     only_if: *not_tag_branch_build_docs
+    skip: $CI == $CI
     depends_on:
         - build
         - local_system_test

Makefile

@@ -591,7 +591,7 @@ test: localunit localintegration remoteintegration localsystem remotesystem  ##
 .PHONY: ginkgo-run
 ginkgo-run: .install.ginkgo
 	$(GINKGO) version
-	$(GINKGO) -vv $(TESTFLAGS) --tags "$(TAGS) remote" $(GINKGOTIMEOUT) --flake-attempts 3 --trace --no-color \
+	$(GINKGO) -vv $(TESTFLAGS) --tags "$(TAGS) remote" $(GINKGOTIMEOUT) --trace --no-color \
 		$(GINKGO_JSON) $(if $(findstring y,$(GINKGO_PARALLEL)),-p,) $(if $(FOCUS),--focus "$(FOCUS)",) \
 		$(if $(FOCUS_FILE),--focus-file "$(FOCUS_FILE)",) $(GINKGOWHAT) $(HACK)
 

go.mod

@@ -20,7 +20,7 @@ require (
 	github.com/containers/libhvee v0.4.1-0.20230920190832-6ab399cadb68
 	github.com/containers/ocicrypt v1.1.8
 	github.com/containers/psgo v1.8.0
-	github.com/containers/storage v1.50.2
+	github.com/containers/storage v1.50.3-0.20230927102810-f969739724eb
 	github.com/coreos/go-systemd/v22 v22.5.0
 	github.com/coreos/stream-metadata-go v0.4.3
 	github.com/crc-org/vfkit v0.1.2-0.20230829083117-09e62065eb6e
@@ -137,7 +137,7 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jinzhu/copier v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
-	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/klauspost/compress v1.17.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
 	github.com/klauspost/pgzip v1.2.6 // indirect
 	github.com/kr/fs v0.1.0 // indirect

go.sum

@@ -271,8 +271,8 @@ github.com/containers/ocicrypt v1.1.8/go.mod h1:jM362hyBtbwLMWzXQZTlkjKGAQf/BN/L
 github.com/containers/psgo v1.8.0 h1:2loGekmGAxM9ir5OsXWEfGwFxorMPYnc6gEDsGFQvhY=
 github.com/containers/psgo v1.8.0/go.mod h1:T8ZxnX3Ur4RvnhxFJ7t8xJ1F48RhiZB4rSrOaR/qGHc=
 github.com/containers/storage v1.43.0/go.mod h1:uZ147thiIFGdVTjMmIw19knttQnUCl3y9zjreHrg11s=
-github.com/containers/storage v1.50.2 h1:Fys4BjFUVNRBEXlO70hFI48VW4EXsgnGisTpk9tTMsE=
-github.com/containers/storage v1.50.2/go.mod h1:dpspZsUrcKD8SpTofvKWhwPDHD0MkO4Q7VE+oYdWkiA=
+github.com/containers/storage v1.50.3-0.20230927102810-f969739724eb h1:Brq+nXVe+vgzJvSqEkRdNb1+FxJMAYLmC1Sm3rQB4HY=
+github.com/containers/storage v1.50.3-0.20230927102810-f969739724eb/go.mod h1:HZESuTLIRmcs00JFSZr6daHD/B51J0ZCZr0T7uDDc9Y=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
@@ -676,8 +676,8 @@ github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdY
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/klauspost/compress v1.15.7/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
-github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
+github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
 github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=

libpod/logs/log.go

@@ -83,6 +83,17 @@ func GetLogFile(path string, options *LogOptions) (*tail.Tail, []*LogLine, error
 	return t, logTail, err
 }
 
+func printFileToStderr(path string) {
+	f, err := os.Open(path)
+	if err != nil {
+		logrus.Errorf("failed to read file: %v", err)
+	}
+	_, err = io.Copy(os.Stderr, f)
+	if err != nil {
+		logrus.Errorf("failed to print file: %v", err)
+	}
+}
+
 func getTailLog(path string, tail int) ([]*LogLine, error) {
 	var (
 		nllCounter int
@@ -120,7 +131,8 @@ func getTailLog(path string, tail int) ([]*LogLine, error) {
 			}
 			nll, err := NewLogLine(lines[i])
 			if err != nil {
-				return nil, err
+				printFileToStderr(path)
+				return nil, fmt.Errorf("parse log line: %w", err)
 			}
 			if !nll.Partial() || first {
 				nllCounter++
@@ -146,7 +158,8 @@ func getTailLog(path string, tail int) ([]*LogLine, error) {
 			if leftover != "" && nllCounter < tail {
 				nll, err := NewLogLine(leftover)
 				if err != nil {
-					return nil, err
+					printFileToStderr(path)
+					return nil, fmt.Errorf("parse leftover log line: %w", err)
 				}
 				tailLog = append(tailLog, nll)
 			}

test/e2e/build_test.go

@@ -864,6 +864,10 @@ RUN ls /dev/test1`, CITEST_IMAGE)
 		session.WaitWithDefaultTimeout()
 		Expect(session).Should(ExitCleanly())
 
+		// 2023-06-21 ESM FIXME
+		ls := SystemExec("bash", []string{"-c", fmt.Sprintf("find %s/events -type f | while read f;do echo;ls -l $f;cat $f;done", tempdir)})
+		ls.WaitWithDefaultTimeout()
+
 		session = podmanTest.Podman([]string{"system", "reset", "-f"})
 		session.WaitWithDefaultTimeout()
 		Expect(session).Should(ExitCleanly())

test/system/helpers.bash

@@ -148,12 +148,16 @@ function basic_setup() {
     for line in "${lines[@]}"; do
         set $line
         echo "# setup(): removing stray external container $1 ($2)" >&3
-        run_podman '?' rm -f $1
+        run_podman '?' --log-level debug rm -f $1
         if [[ $status -ne 0 ]]; then
             echo "# [setup] $_LOG_PROMPT podman rm -f $1" >&3
             for errline in "${lines[@]}"; do
                 echo "# $errline" >&3
             done
+            echo "# PS OUTPUT" >&3
+            ps aux | while read i; do echo "# $i" >&3; done
+            echo "# MOUNTINFO" >&3
+            cat /proc/self/mountinfo | while read i; do echo "# $i" >&3; done
             # FIXME FIXME FIXME: temporary hack for #18831. If we see the
             # unmount/EINVAL flake, nothing will ever work again.
             if [[ $output =~ unmounting.*invalid ]]; then

test/utils/matchers.go

@@ -179,7 +179,7 @@ type exitCleanlyMatcher struct {
 
 type podmanSession interface {
 	ExitCode() int
-	ErrorToString() string
+	ErrContents() []byte
 }
 
 func (matcher *exitCleanlyMatcher) Match(actual interface{}) (success bool, err error) {
@@ -189,10 +189,10 @@ func (matcher *exitCleanlyMatcher) Match(actual interface{}) (success bool, err
 	}
 
 	exitcode := session.ExitCode()
-	stderr := session.ErrorToString()
+	stderr := session.ErrContents()
 	if exitcode != 0 {
 		matcher.msg = fmt.Sprintf("Command failed with exit status %d", exitcode)
-		if stderr != "" {
+		if len(stderr) != 0 {
 			matcher.msg += ". See above for error message."
 		}
 		return false, nil
@@ -214,8 +214,8 @@ func (matcher *exitCleanlyMatcher) Match(actual interface{}) (success bool, err
 		return true, nil
 	}
 
-	if stderr != "" {
-		matcher.msg = fmt.Sprintf("Unexpected warnings seen on stderr: %q", stderr)
+	if len(stderr) != 0 {
+		matcher.msg = fmt.Sprintf("Unexpected warnings seen on stderr: %q", string(stderr))
 		return false, nil
 	}