Merge pull request #710 from jbenet/sharness-gateway-fix

sharness fixes - gateway and more

cmd/ipfs/daemon.go

@@ -183,6 +183,7 @@ func daemonFunc(req cmds.Request, res cmds.Response) {
 			if rootRedirect != nil {
 				opts = append(opts, rootRedirect)
 			}
+			fmt.Printf("Gateway server listening on %s\n", gatewayMaddr)
 			err := corehttp.ListenAndServe(node, gatewayMaddr.String(), opts...)
 			if err != nil {
 				log.Error(err)
@@ -198,6 +199,7 @@ func daemonFunc(req cmds.Request, res cmds.Response) {
 	if rootRedirect != nil {
 		opts = append(opts, rootRedirect)
 	}
+	fmt.Printf("API server listening on %s\n", apiMaddr)
 	if err := corehttp.ListenAndServe(node, apiMaddr.String(), opts...); err != nil {
 		res.SetError(err, cmds.ErrNormal)
 		return

core/corehttp/corehttp.go

@@ -1,7 +1,6 @@
 package corehttp
 
 import (
-	"fmt"
 	"net/http"
 
 	manners "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/braintree/manners"
@@ -36,10 +35,10 @@ func ListenAndServe(n *core.IpfsNode, listeningMultiAddr string, options ...Serv
 			return err
 		}
 	}
-	return listenAndServe("API", n, addr, mux)
+	return listenAndServe(n, addr, mux)
 }
 
-func listenAndServe(name string, node *core.IpfsNode, addr ma.Multiaddr, mux *http.ServeMux) error {
+func listenAndServe(node *core.IpfsNode, addr ma.Multiaddr, mux *http.ServeMux) error {
 	_, host, err := manet.DialArgs(addr)
 	if err != nil {
 		return err
@@ -52,7 +51,6 @@ func listenAndServe(name string, node *core.IpfsNode, addr ma.Multiaddr, mux *ht
 	serverExited := make(chan struct{})
 
 	go func() {
-		fmt.Printf("%s server listening on %s\n", name, addr)
 		serverError = server.ListenAndServe(host, mux)
 		close(serverExited)
 	}()

test/sharness/Makefile

@@ -26,7 +26,7 @@ aggregate:
 	@echo "*** $@ ***"
 	lib/test-aggregate-results.sh
 
-deps: $(SHARNESS) $(BINS)
+deps: $(SHARNESS) $(BINS) curl
 
 $(SHARNESS):
 	@echo "*** installing $@ ***"
@@ -37,3 +37,9 @@ bin/%: $(IPFS_ROOT)/**/*.go
 	cd .. && make $@
 
 .PHONY: all clean $(T) aggregate
+
+# will fail if curl is not installed.
+# TODO: get rid of this and install curl with git or ipfs.
+install_curl_pls = "curl is required to run tests. please install it"
+curl:
+	@which curl >/dev/null || (echo "$(install_curl_pls)" && false)

test/sharness/lib/test-lib-hashes.sh

@@ -0,0 +1,5 @@
+# this file defines several useful hashes used across the test codebase.
+# thus they can be defined + changed in one place
+
+HASH_WELCOME_DOCS="QmPXME1oRtoT627YKaDPDQ3PwA8tdP9rWuAAweLzqSwAWT"
+HASH_EMPTY_DIR="QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn"

test/sharness/lib/test-lib.sh

@@ -23,6 +23,11 @@ if test `which ipfs` != $(pwd)/bin/ipfs; then
 	exit 1
 fi
 
+
+# source the common hashes first.
+. lib/test-lib-hashes.sh
+
+
 SHARNESS_LIB="lib/sharness/sharness.sh"
 
 . "$SHARNESS_LIB" || {
@@ -40,9 +45,16 @@ test_cmp() {
 
 # Please put go-ipfs specific shell functions below
 
+# grab + output options
 test "$TEST_NO_FUSE" != 1 && test_set_prereq FUSE
 test "$TEST_EXPENSIVE" = 1 && test_set_prereq EXPENSIVE
 
+if test "$TEST_VERBOSE" = 1; then
+	echo '# TEST_VERBOSE='"$TEST_VERBOSE"
+	echo '# TEST_NO_FUSE='"$TEST_NO_FUSE"
+	echo '# TEST_EXPENSIVE='"$TEST_EXPENSIVE"
+fi
+
 test_cmp_repeat_10_sec() {
 	for i in 1 2 3 4 5 6 7 8 9 10
 	do
@@ -52,6 +64,15 @@ test_cmp_repeat_10_sec() {
 	test_cmp "$1" "$2"
 }
 
+test_run_repeat_10_sec() {
+	for i in 1 2 3 4 5 6 7 8 9 10
+	do
+		(test_eval_ "$1") && return
+		sleep 1
+	done
+	return 1 # failed
+}
+
 test_wait_output_n_lines_60_sec() {
 	echo "$2" >expected_waitn
 	for i in 1 2 3 4 5 6 7 8 9 10
@@ -66,7 +87,11 @@ test_wait_output_n_lines_60_sec() {
 
 test_wait_open_tcp_port_10_sec() {
 	for i in 1 2 3 4 5 6 7 8 9 10; do
-		if [ $(ss -lt "sport == :$1" | wc -l) -gt 1 ]; then
+		# this is not a perfect check, but it's portable.
+		# cant count on ss. not installed everywhere.
+		# cant count on netstat using : or . as port delim. differ across platforms.
+		echo $(netstat -aln | egrep "^tcp.*LISTEN" | egrep "[.:]$1" | wc -l) -gt 0
+		if [ $(netstat -aln | egrep "^tcp.*LISTEN" | egrep "[.:]$1" | wc -l) -gt 0 ]; then
 			return 0
 		fi
 		sleep 1
@@ -90,19 +115,49 @@ test_init_ipfs() {
 
 }
 
+test_config_ipfs_gateway_readonly() {
+	test_expect_success "prepare config -- gateway readonly" '
+	  ipfs config Addresses.Gateway /ip4/0.0.0.0/tcp/5002
+	'
+}
+
+test_config_ipfs_gateway_writable() {
+	test_expect_success "prepare config -- gateway writable" '
+	  ipfs config Addresses.Gateway /ip4/0.0.0.0/tcp/5002 &&
+	  ipfs config -bool Gateway.Writable true
+	'
+}
+
 test_launch_ipfs_daemon() {
 
 	test_expect_success "'ipfs daemon' succeeds" '
 		ipfs daemon >actual_daemon 2>daemon_err &
 	'
 
-	test_expect_success "'ipfs daemon' output looks good" '
+	# we say the daemon is ready when the API server is ready.
+	# and we make sure there are no errors
+	test_expect_success "'ipfs daemon' is ready" '
 		IPFS_PID=$! &&
-		echo "API server listening on /ip4/127.0.0.1/tcp/5001" >expected_daemon &&
-		test_cmp_repeat_10_sec expected_daemon actual_daemon ||
-		fsh cat daemon_err
+		test_run_repeat_10_sec "cat actual_daemon | grep \"API server listening on\"" &&
+		printf "" >empty && test_cmp daemon_err empty ||
+		fsh cat actual_daemon || fsh cat daemon_err
+	'
+
+	ADDR_API="/ip4/127.0.0.1/tcp/5001"
+	test_expect_success "'ipfs daemon' output includes API address" '
+		cat actual_daemon | grep "API server listening on $ADDR_API" ||
+		fsh cat actual_daemon ||
+		fsh "cat actual_daemon | grep \"API server listening on $ADDR_API\""
 	'
 
+	ADDR_GWAY=`ipfs config Addresses.Gateway`
+	if test "$ADDR_GWAY" != ""; then
+		test_expect_success "'ipfs daemon' output includes Gateway address" '
+			cat actual_daemon | grep "Gateway server listening on $ADDR_GWAY" ||
+			fsh cat actual_daemon ||
+			fsh "cat actual_daemon | grep \"Gateway server listening on $ADDR_GWAY\""
+		'
+	fi
 }
 
 test_mount_ipfs() {

test/sharness/t0020-init.sh

@@ -10,7 +10,8 @@ test_description="Test init command"
 
 test_expect_success "ipfs init succeeds" '
 	export IPFS_PATH="$(pwd)/.go-ipfs" &&
-	ipfs init >actual_init
+	BITS="2048" &&
+	ipfs init --bits="$BITS" >actual_init
 '
 
 test_expect_success ".go-ipfs/ has been created" '
@@ -34,10 +35,9 @@ test_expect_success "ipfs peer id looks good" '
 '
 
 test_expect_success "ipfs init output looks good" '
-	STARTHASH="QmPXME1oRtoT627YKaDPDQ3PwA8tdP9rWuAAweLzqSwAWT" &&
-	STARTFILE="ipfs cat /ipfs/$STARTHASH/readme"
+	STARTFILE="ipfs cat /ipfs/$HASH_WELCOME_DOCS/readme" &&
 	echo "initializing ipfs node at $IPFS_PATH" >expected &&
-	echo "generating 4096-bit RSA keypair...done" >>expected &&
+	echo "generating $BITS-bit RSA keypair...done" >>expected &&
 	echo "peer identity: $PEERID" >>expected &&
 	echo "to get started, enter:" >>expected &&
 	printf "\\n\\t$STARTFILE\\n\\n" >>expected &&

test/sharness/t0060-daemon.sh

@@ -34,11 +34,12 @@ test_expect_success "ipfs peer id looks good" '
 
 # note this is almost the same as t0020-init.sh "ipfs init output looks good"
 test_expect_success "ipfs daemon output looks good" '
-  STARTHASH="QmTTFXiXoixwT53tcGPu419udsHEHYu6AHrQC8HAKdJYaZ" &&
+  STARTFILE="ipfs cat /ipfs/$HASH_WELCOME_DOCS/readme" &&
   echo "initializing ipfs node at $IPFS_PATH" >expected &&
-  echo "generating key pair...done" >>expected &&
+  echo "generating 4096-bit RSA keypair...done" >>expected &&
   echo "peer identity: $PEERID" >>expected &&
-  echo "\nto get started, enter: ipfs cat $STARTHASH" >>expected &&
+  echo "to get started, enter:" >>expected &&
+  printf "\\n\\t$STARTFILE\\n\\n" >>expected &&
   echo "daemon listening on /ip4/127.0.0.1/tcp/5001" >>expected &&
   test_cmp_repeat_10_sec expected actual_init
 '

test/sharness/t0080-repo.sh

@@ -43,8 +43,8 @@ test_expect_success "'ipfs pin rm' succeeds" '
 
 test_expect_success "file no longer pinned" '
 	# we expect the welcome files to show up here
-	echo QmPXME1oRtoT627YKaDPDQ3PwA8tdP9rWuAAweLzqSwAWT >expected2 &&
-	ipfs refs -r QmPXME1oRtoT627YKaDPDQ3PwA8tdP9rWuAAweLzqSwAWT >>expected2 &&
+	echo "$HASH_WELCOME_DOCS" >expected2 &&
+	ipfs refs -r "$HASH_WELCOME_DOCS" >>expected2 &&
 	cat expected2 | sort >expected_sorted2 &&
 	ipfs pin ls -type=recursive | sort >actual2 &&
 	test_cmp expected_sorted2 actual2
@@ -87,8 +87,8 @@ test_expect_success "'ipfs repo gc' removes file" '
 
 test_expect_success "'ipfs refs local' no longer shows file" '
 	echo QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn >expected8 &&
-	echo QmPXME1oRtoT627YKaDPDQ3PwA8tdP9rWuAAweLzqSwAWT >>expected8 &&
-	ipfs refs -r QmPXME1oRtoT627YKaDPDQ3PwA8tdP9rWuAAweLzqSwAWT >>expected8 &&
+	echo "$HASH_WELCOME_DOCS" >>expected8 &&
+	ipfs refs -r "$HASH_WELCOME_DOCS" >>expected8 &&
 	cat expected8 | sort >expected_sorted8 &&
 	ipfs refs local | sort >actual8 &&
 	test_cmp expected_sorted8 actual8
@@ -101,7 +101,7 @@ test_expect_success "adding multiblock random file succeeds" '
 
 test_expect_success "'ipfs pin ls -type=indirect' is correct" '
 	ipfs refs "$MBLOCKHASH" >refsout &&
-	ipfs refs -r "QmPXME1oRtoT627YKaDPDQ3PwA8tdP9rWuAAweLzqSwAWT" >>refsout &&
+	ipfs refs -r "$HASH_WELCOME_DOCS" >>refsout &&
 	cat refsout | sort >refsout_sorted &&
 	ipfs pin ls -type=indirect | sort >indirectpins &&
 	test_cmp refsout_sorted indirectpins
@@ -129,8 +129,8 @@ test_expect_success "'ipfs pin ls -type=direct' is correct" '
 
 test_expect_success "'ipfs pin ls -type=recursive' is correct" '
 	echo "$MBLOCKHASH" >rp_expected &&
-	echo QmPXME1oRtoT627YKaDPDQ3PwA8tdP9rWuAAweLzqSwAWT >>rp_expected &&
-	ipfs refs -r "QmPXME1oRtoT627YKaDPDQ3PwA8tdP9rWuAAweLzqSwAWT" >>rp_expected &&
+	echo "$HASH_WELCOME_DOCS" >>rp_expected &&
+	ipfs refs -r "$HASH_WELCOME_DOCS" >>rp_expected &&
 	cat rp_expected | sort >rp_exp_sorted &&
 	ipfs pin ls -type=recursive | sort >rp_actual &&
 	test_cmp rp_exp_sorted rp_actual

test/sharness/t0110-gateway.sh

@@ -9,8 +9,16 @@ test_description="Test HTTP gateway"
 . lib/test-lib.sh
 
 test_init_ipfs
+test_config_ipfs_gateway_readonly
 test_launch_ipfs_daemon
 
+# TODO check both 5001 and 5002.
+# 5001 should have a readable gateway (part of the API)
+# 5002 should have a readable gateway (using ipfs config Addresses.Gateway)
+# but ideally we should only write the tests once. so maybe we need to
+# define a function to test a gateway, and do so for each port.
+# for now we check 5001 here as 5002 will be checked in gateway-writable.
+
 test_expect_success "GET IPFS path succeeds" '
   echo "Hello Worlds!" > expected &&
   HASH=`ipfs add -q expected` &&