Skip to content

Commit

Permalink
fix
Browse files Browse the repository at this point in the history
Signed-off-by: Ryan Leung <rleungx@gmail.com>
  • Loading branch information
rleungx committed Feb 20, 2025
1 parent 4172b92 commit dbc4fc3
Show file tree
Hide file tree
Showing 2 changed files with 29 additions and 70 deletions.
81 changes: 23 additions & 58 deletions tests/integrations/realcluster/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,8 @@ tidy:
git diff go.mod go.sum | cat
git diff --quiet go.mod go.sum

.PHONY: check
check: tiup test

.PHONY: tiup
tiup:
@echo "==> Checking tiup installation"
@if ! which tiup > /dev/null 2>&1; then \
Expand All @@ -42,71 +40,38 @@ tiup:
fi
@echo "tiup version: $$(tiup --version)"

.PHONY: deploy
deploy: kill_cluster deploy_only

.PHONY: deploy_only
deploy_only:
@echo "==> Deploying cluster..."
@./deploy.sh
@echo "==> Waiting for cluster to be ready..."
@./wait_tiup.sh 15 20
@echo "==> Checking cluster status..."
@if pid=$$(pgrep -f "tiup.*playground.*--tag"); then \
echo "Playground process running with pid: $$pid"; \
tiup playground display; \
else \
echo "ERROR: Playground process not found"; \
exit 1; \
fi
@ echo "deploying..."
./deploy.sh
@ echo "wait cluster ready..."
./wait_tiup.sh 15 20
@ echo "check cluster status..."
@ pid=$$(ps -ef | grep 'playground' | grep -v grep | awk '{print $$2}' | head -n 1); \
echo $$pid;

.PHONY: kill_cluster
kill_cluster:
@echo "==> Killing cluster..."
@if pid=$$(pgrep -f "tiup.*playground.*--tag"); then \
echo "Found playground process(es):"; \
ps -fp $$pid; \
echo "Sending SIGTERM..."; \
@ echo "kill cluster..."
@ pid=$$(ps -ef | grep 'playground' | grep -v grep | awk '{print $$2}' | head -n 1); \
if [ ! -z "$$pid" ]; then \
echo $$pid; \
kill $$pid; \
echo "Waiting for process to exit (30s)..."; \
for i in $$(seq 1 30); do \
if ! kill -0 $$pid 2>/dev/null; then \
echo "Process exited"; \
exit 0; \
fi; \
sleep 1; \
done; \
echo "Process still running, sending SIGKILL..."; \
kill -9 $$pid || true; \
else \
echo "No playground process found"; \
echo "waiting for cluster to exit..."; \
sleep 30; \
fi

.PHONY: test
test:
@echo "==> Running integration tests..."
@CGO_ENABLED=1 go test ./... -v -tags deadlock -race -cover || (\
echo "==> Test failed. Collecting logs..."; \
$(MAKE) collect_logs; \
exit 1 \
)

.PHONY: collect_logs
collect_logs:
@echo "==> Collecting cluster logs..."
@for pd in 0 1 2; do \
echo "### PD-$$pd logs ###"; \
cat ~/.tiup/data/pd_real_cluster_test/pd-$$pd/pd.log 2>/dev/null || echo "No log file found"; \
echo; \
done
@echo "==> Collecting playground logs..."
@find /tmp/real_cluster/playground -type f -name "*.log" -exec sh -c 'echo "### $$(basename {}) ###"; cat {}; echo' \;

.PHONY: clean
clean:
@echo "==> Cleaning up..."
@rm -rf /tmp/real_cluster/playground/*
@echo "==> Cleaned playground logs"
CGO_ENABLED=1 go test ./... -v -tags deadlock -race -cover || (\
echo "The following is pd-0 log" ; \
cat ~/.tiup/data/pd_real_cluster_test/pd-0/pd.log ; \
echo "The following is pd-1 log" ; \
cat ~/.tiup/data/pd_real_cluster_test/pd-1/pd.log ; \
echo "The following is pd-2 log" ; \
cat ~/.tiup/data/pd_real_cluster_test/pd-2/pd.log ; \
exit 1)

install-tools:
cd $(ROOT_PATH) && $(MAKE) install-tools

.PHONY: tidy static check tiup deploy deploy_only kill_cluster test install-tools
18 changes: 6 additions & 12 deletions tests/integrations/realcluster/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -143,12 +143,13 @@ func (c *cluster) stop() {
_ = syscall.Kill(pid, syscall.SIGKILL)
}
}
log.Info("cluster destroyed", zap.String("tag", c.tag))
log.Info("cluster stopped", zap.String("tag", c.tag))
}

func (c *cluster) deploy() {
re := c.re
curPath, err := os.Getwd()
log.Info(curPath)
re.NoError(err)
re.NoError(os.Chdir("../../.."))

Expand Down Expand Up @@ -218,18 +219,13 @@ func (c *cluster) collectPids() error {

func (c *cluster) waitReady() {
re := c.re
const (
interval = 5
maxTimes = 20
)
log.Info("start to wait TiUP ready", zap.String("tag", c.tag))
timeout := time.After(time.Duration(maxTimes*interval) * time.Second)
ticker := time.NewTicker(time.Duration(interval) * time.Second)
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()

for i := range maxTimes {
for {
select {
case <-timeout:
case <-time.After(100 * time.Second):
re.FailNowf("TiUP is not ready after timeout, tag: %s", c.tag)
case <-ticker.C:
log.Info("check TiUP ready", zap.String("tag", c.tag))
Expand All @@ -240,11 +236,9 @@ func (c *cluster) waitReady() {
return
}
log.Info(output)
log.Info("TiUP is not ready, will retry", zap.Int("retry times", i),
zap.String("tag", c.tag), zap.Error(err))
log.Info("TiUP is not ready, will retry", zap.String("tag", c.tag), zap.Error(err))
}
}
re.FailNowf("TiUP is not ready after max retries, tag: %s", c.tag)
}

func buildBinPathsOpts(ms bool) string {
Expand Down

0 comments on commit dbc4fc3

Please sign in to comment.