curl.rexx

/*                                                                    *
 *        Name: WGET REXX or CURL REXX                                *
 *        Date: 2006-Mar-02                                           *
 *              This program is part of the CMS Make package.         *
 *       Calls: WEBROVER REXX when it cannot handle a given URL.      *
 *                                                                    */

Numeric Digits 20

make_version = "2.0.35"

/* if no other output, attach console */
'STREAMSTATE OUTPUT'
If rc = 4 Then rc = 0
If rc = 8 Then rc = 0
If rc = 12 Then 'ADDPIPE *.OUTPUT: | CONSOLE'
If rc /= 0 Then Exit rc

verbose = 0
outfile = ""
trans = 1

/* parse command-line and options */
Parse Source cms cmd argu . . arg0 .
Parse Arg args
Parse Var args arg1 .
Do While Left(arg1,1) = "-"
  Parse Var args . args
  Select
    /* shorter option nomenclature may cange between CURL and WGET */
    When argu = "CURL" & Abbrev("-s",arg1,2) Then verbose = 0
    When argu = "CURL" & Abbrev("-o",arg1,2) Then Parse Var args outfile args
    When argu = "WGET" & Abbrev("-q",arg1,2) Then verbose = 0
    When argu = "WGET" & Abbrev("-nv",arg1,3) Then verbose = 0
    When argu = "WGET" & Abbrev("-O",arg1,2) Then Parse Var args outfile args
    When   /* both */    Abbrev("-v",arg1,2) Then verbose = 1
    When Abbrev("--silent",arg1,8) Then verbose = 0
    When Abbrev("--non-verbose",arg1,13) Then verbose = 0
    When Abbrev("--verbose",arg1,9) Then verbose = 1
    When Abbrev("--binary",arg1,5) Then trans = 0
    When Abbrev("--use-binary",arg1,12) Then trans = 0
    When Abbrev("--ascii",arg1,5) Then trans = 1
    When Abbrev("--use-ascii",arg1,11) Then trans = 1
    When Abbrev("-B",arg1,2) Then trans = 1

    When Abbrev("--text",arg1,6) Then trans = 1
    When Abbrev("--output",arg1,8) Then Parse Var args outfile args
    When Abbrev("-h",arg1,2) | Abbrev("--help",arg1,6) Then Do
      'CALLPIPE COMMAND HELP CMS' argu '(NOSCREEN' ,
        '| NLOCATE 1.10 /DMSHLL355I/ | UNIQ | *.OUTPUT:'
      Exit rc
    End /* When .. Do */
    When Abbrev("--version",arg1,9) Then Do
      'CALLPIPE VAR MAKE_VERSION' ,
        '| SPEC /CMS Make/ NW W 1 NW /' || argu || '/ NW | *.OUTPUT:'
      Exit 0
    End /* When .. Do */

    When Abbrev("--insecure",arg1,10) ,
       | Abbrev("--no-check-certificate",arg1,22) ,
       | Abbrev("--cacert",arg1,8) ,
       | Abbrev("--capath",arg1,8) ,
       | Abbrev("--cert",arg1,6) ,
       | Abbrev("-E",arg1,2) ,
       | Abbrev("-k",arg1,2) Then Do
      Address "COMMAND" 'XMITMSG 14 ARG1 (ERRMSG CALLER URL'
      Say "CMS Pipelines needs SSL support"
      Exit 24
    End

    Otherwise Do
      Address "COMMAND" 'XMITMSG 3 ARG1 (ERRMSG CALLER URL'
      Exit 24
    End /* Otherwise Do */
  End /* Select */
  Parse Var args arg1 .
End /* Do While */
If argu = "CURL" & outfile = "" Then outfile = "-"

/* load standard ASCII <--> EBCDIC translation */
'CALLPIPE STATE POSIX TCPXLBIN'
If rc = 28 Then Address "COMMAND" 'EXEC VMLINK TCPMAINT 592'
'CALLPIPE < POSIX TCPXLBIN | STEM TCP.'
If rc /= 0 Then Return rc ;  a2e = tcp.2 ;  e2a = tcp.3

/* get essential config info - like the stack name */
'CALLPIPE < TCPIP DATA | STRIP' ,
  '| NLOCATE 1.1 /;/ | STEM TCF.'
If rc /= 0 Then Return rc
tcp.id = "TCPIP"
Do i = 1 to tcf.0
  Parse Upper Var tcf.i cf c1 .
  Select
    When Abbrev("TCPIPUSERID",cf,6) Then tcp.id = c1
    Otherwise nop
  End /* Select */
End /* Do For */

/* step through command line list-o-URLs */
Do While args /= ""
  Parse Var args furl args
  Parse Var furl mode '://' host '/' file
  port = ""
  user = ""
  pass = ""
  If POS('@',host) > 0 Then Parse Var host user '@' host
  If POS(':',host) > 0 Then Parse Var host host ':' port
  If POS(':',user) > 0 Then Parse Var user user ':' pass
  Upper mode
  file = "/" || file
  If verbose Then Do
    Say "PROTOCOL" mode
    Say "USERNAME" user
    Say "PASSWORD" pass
    Say "    HOST" host
    Say "    PORT" port
    Say "FILENAME" file
  End /* If .. Do */

  /* CALL OUT HERE */
  Select /* mode */
    When mode = "HTTP" Then Parse Value ,
      url_http(user,pass,host,port,file,outfile,trans) With rc rs
    When mode = "FTP" Then Parse Value ,
      url_ftp(user,pass,host,port,file,outfile,trans) With rc rs
    When mode = "FILE" Then Parse Value ,
      url_file(user,pass,host,port,file,outfile,trans) With rc rs
    Otherwise Do
/*    Address "COMMAND" 'XMITMSG 15 MODE (ERRMSG CALLER URL'          */
      Address "COMMAND" 'XMITMSG 475 MODE (ERRMSG CALLER URL'
/*    Address "COMMAND" 'XMITMSG 2481 MODE (ERRMSG CALLER URL'        */
      Exit 24 ;  End
  End /* Select */

  If rc /= 0 Then Exit rc

End

Exit

/* ------------------------------------------------------------ URL_HTTP
 *  This routine is for files fetched via HTTP.
 */
url_http: Procedure Expose tcp. ;  a2e = tcp.2 ;  e2a = tcp.3
Parse Arg user,pass,host,port,file,outfile,trans,.
Parse Var file file ";" flag      /* look for ";type=A" to mean ASCII */
Upper flag
If flag = "TYPE=A" Then trans = 1

/* add a stream for HTTP header handling if not already there */
Trace Off
'STREAMSTATE INPUT HTTP'
If rc = -4 Then 'ADDSTREAM HTTP'
Trace Normal

If Verify(host,"0123456789.") > 0 Then Do
  /* 'CALLPIPE VAR HOST | HOSTBYNAME | VAR ADDR' */
  /* a fix from phsiii */
  'CALLPIPE VAR HOST | HOSTBYNAME | SPECS w1 1 | VAR ADDR'
  If rc /= 0 Then Return rc
End /* If .. Do */
Else addr = host

If port = "" Then port = 80

If outfile = "" Then outfile = basename(file)

/* determine output to file or stream */
Parse Upper Var outfile fn '.' ft '.' .
If fn = "" Then fn = Userid()
If ft = "" & trans Then ft = "TXT"
If ft = "" Then ft = "BIN"
If fn = "-" Then pipe = "*.OUTPUT:"
            Else pipe = 'PAD 1 | >' fn ft 'A'

/* build a proper HTTP request */
req.1 = "GET" file "HTTP/1.0"
req.2 = "Host:" host
req.3 = "Accept: */*"
req.4 = ""
req.0 = 4

/* now drive it either with or without translation */
a2ex = "X" || C2X(a2e)
e2ax = "X" || C2X(e2a)
'ADDPIPE' ,
    '*.OUTPUT.HTTP: | TCPCLIENT' addr port ,
                      'LINGER 7 USERID' tcp.id 'DEBLOCK LINEND 0A' ,
                   '| *.INPUT.HTTP:'
If rc /= 0 Then Return rc

/* send the request header then read the response header */
'CALLPIPE (END !) STEM REQ.' ,
  '| E2A: XLATE | SPEC 1-* 1 x0D0A N' ,
  '| *.OUTPUT.HTTP:',
  '! STRLITERAL' e2ax '| E2A:'
If rc /= 0 Then Return rc

/* consume the response header */
'SELECT INPUT HTTP'
size = 0
i = 0
lm = ""
Do Forever
  'PEEKTO RECORD'
  If rc /= 0 Then Leave
  Parse Var record record '0D'x
  If record = "" Then Leave
  i = i + 1 ; h.i = Translate(record,a2e)
  Parse Upper Var h.i key val .
  If key = "CONTENT-LENGTH:" Then size = val
  If key = "LAST-MODIFIED:" Then lm = h.i
  'READTO'
End
If rc = 12 Then rc = 0
If rc /= 0 Then Do
  _rc = rc
  'SELECT INPUT 0'
  Return _rc
End
h.0 = i
'READTO'
/* Say "size indicated is" size */

/* check the HTTP return code */
If h.0 > 0 Then Do
  Parse Var h.1 . _rc rs
  If _rc = 200 Then _rc = 0
  If _rc /= 0 Then Do
    'SELECT INPUT 0'
    Return _rc rs
  End
End

/* write the content to file or to following stage */
If trans Then 'CALLPIPE (END !) *.INPUT.HTTP:' ,
  '| STRIP TRAILING x0D | A2E: XLATE | CHANGE xB0 x5F' ,
  '|' pipe ,
  '! STRLITERAL' a2ex  '| A2E:'
         Else 'CALLPIPE         *.INPUT.HTTP:' ,
  '| BLOCK 512 LINEND 0A' ,
  '|' pipe
_rc = rc
'SEVER INPUT' ; 'SEVER OUTPUT'

/* switch back to primary input stream */
'SELECT INPUT 0'

/* 'CALLPIPE STEM H. | > WGET HEAD A' */
/* stamp time on file if a file was written */
If fn /= "-" & lm /= "" Then lm = _lm2full(lm)
If fn /= "-" & lm /= "" Then Address "COMMAND" 'DMSPLU' fn ft 'A' lm

Return _rc

/* ------------------------------------------------------------- URL_FTP
 *  This routine is for files fetched using FTP.
 *  We punt to VM TCP/IP FTP utility wrapping it in a 'MAKEBUF'.
 */
url_ftp: Procedure Expose tcp. ;  a2e = tcp.2 ;  e2a = tcp.3
Parse Arg user,pass,host,port,file,outfile,trans,.
Parse Var file file ";" flag      /* look for ";type=A" to mean ASCII */
Upper flag
If flag = "TYPE=A" Then trans = 1

If Verify(host,"0123456789.") > 0 Then Do
  /* 'CALLPIPE VAR HOST | HOSTBYNAME | VAR ADDR' */
  /* a fix from phsiii */
  'CALLPIPE VAR HOST | HOSTBYNAME | SPECS w1 1 | VAR ADDR'
  If rc /= 0 Then Return rc
End /* If .. Do */
Else addr = host

If outfile = "" Then outfile = basename(file)

/* determine output to file or stream */
Parse Upper Var outfile fn '.' ft '.' .
If fn = "" Then fn = Userid()
If ft = "" & trans Then ft = "TXT"
If ft = "" Then ft = "BIN"
If fn = "-" Then pipe = "*.OUTPUT:"
            Else pipe = ""

/* build a standard FTP request */
If user = "" Then user = "anonymous"
If pass = "" Then pass = Userid()
If port = "" Then port = 21

/* parse server-side file and directory */
file = Reverse(file)
Parse Var file file '/' dir
dir = Reverse(dir)
file = Reverse(file)

temp = "temp.file"

Address "COMMAND" 'MAKEBUF'

  Queue user pass
  If trans Then Queue "TYPE A"
           Else Queue "TYPE I"
  Do While dir /= ""
    Parse Var dir dir1 "/" dir
    Queue "CD" dir1
  End
  Queue "GET" file temp "(REPLACE"
  Queue "QUIT"

  Address "CMS" 'FTP' host port
  ftprc = rc

Address "COMMAND" 'DROPBUF'

If ftprc /= 0 Then Return ftprc

/* pipeline fixup here */
Parse Upper Var temp tfn '.' tft '.' .
If pipe = "" Then Do
  Address "CMS" 'RENAME' tfn tft "A" fn ft "="
End ;        Else Do
  'CALLPIPE <' tfn tft '|' pipe
  ftprc = rc
  Address "COMMAND" 'ERASE' tfn tft "A"
End

Return ftprc

/* ------------------------------------------------------------ URL_FILE
 *  This routine is for local files.
 */
url_file: Procedure Expose tcp. ;  a2e = tcp.2 ;  e2a = tcp.3
Parse Arg user,pass,host,port,file,outfile,trans,.
Parse Var file file ";" flag      /* look for ";type=A" to mean ASCII */
Upper flag
If flag = "TYPE=A" Then trans = 1

file = Strip(file,"L","/")
Parse Var file fn "." ft "." .
'CALLPIPE <' fn ft '| *.OUTPUT:'

Return rc

/* ------------------------------------------------------------ VIAROVER
 *  For some URLs, just punt to WEBROVER, assuming it is present.
 */
viarover: Procedure Expose tcp. ;  a2e = tcp.2 ;  e2a = tcp.3
Parse Arg f . , o . , t . , .

/* force WEBROVER into binary mode */
If o = "" Then o = basename(f)
f = f || ";binary"
/* this is a sure way to remove HTTP headers from files retrieved */

/* determine output to file or stream */
Parse Upper Var o fn '.' ft '.' .
If fn = "" Then fn = Userid()
If ft = "" & t Then ft = "TXT"
If ft = "" Then ft = "BIN"
If fn = "-" Then pipe = "*.OUTPUT:"
            Else pipe = 'PAD 1 | >' fn ft 'A'

/* now drive it either with or without translation */
If t Then ,
'CALLPIPE (END !) VAR F | WEBROVER | DEBLOCK LINEND 0A' ,
  '| STRIP TRAILING x0D | A2E: XLATE | CHANGE xB0 x5F' ,
  '|' pipe ,
  '! VAR A2E | A2E:'
Else 'CALLPIPE VAR F | WEBROVER |' pipe

Return rc

/* ------------------------------------------------------------ BASENAME
 *  This routine returns the basename of the path supplied.
 */
basename: Procedure
Parse Arg file . , .
file = Reverse(file)
Parse Var file file '/' .
Return Reverse(file)

/* ---------------------------------------------------------------------
 *  Convert Last-Modified HTTP header to FULLDATE format.
 */
_lm2full: Procedure
Parse Arg args

/* parse standard Last-Modified and slice off date and time */
Parse Upper Arg . "," dd mon yyyy time zone . , .
lmdt = dd mon yyyy time

/* convert to POSIX for easier arithmetic */
Address "COMMAND" ,
'PIPE VAR LMDT | DATECONVERT NORMAL POSIX | VAR PDATE'
If rc /= 0 Then Return ""

/* apply time zone offset */
zdate = C2D(pdate) + tzoffset("S")
zdate = D2C(zdate,8)

/* convert back to FULLDATE and include time */
Address "COMMAND" ,
'PIPE VAR ZDATE | DATECONVERT POSIX FULLDATE TIMEOUT | VAR RS'
If rc /= 0 Then Return ""

/* slice off fractional time and return usable stamp */
Parse Var rs rd rt .
Parse Var rt rt "." .
Return rd rt

/* ------------------------------------------------------------ TZOFFSET
 *  Compute timezone offset based on timezone string from 'CP Q TIME'.
 *  (we probably have a CSL routine to do this ... but maybe not)
 */
tzoffset: Procedure

Parse Upper Arg denom . ',' . , .
Parse Upper Value Diag(08,'QUERY TIME') With . . . tz .

Select /* tz */
  When tz = "PST" Then zo = -8
  When tz = "PDT" Then zo = -7
  When tz = "MST" Then zo = -7
  When tz = "MDT" Then zo = -6
  When tz = "CST" Then zo = -6
  When tz = "CDT" Then zo = -5
  When tz = "EST" Then zo = -5
  When tz = "EDT" Then zo = -4
  When tz = "CET"  Then zo = 1
  When tz = "CEDT" Then zo = 2
  When tz = "CEST" Then zo = 2
  When tz = "EET"  Then zo = 2
  When tz = "EEDT" Then zo = 3
  When tz = "EEST" Then zo = 3
  When tz = "WET"  Then zo = 0
  When tz = "WEDT" Then zo = 1
  When tz = "WEST" Then zo = 1
  /* FEED ME: need more timezones, duh */
  When tz = "GMT"  Then zo = 0
  When tz = "UTC"  Then zo = 0
  Otherwise zo = 0
End /* Select tz */

denom = Left(denom,1)
Select /* denom */
  When denom = "S" Then Return zo * 60 * 60      /* offset in seconds */
  When denom = "M" Then Return zo * 60           /* offset in minutes */
  Otherwise Return zo
End /* Select denom */

/*
 * There are many more options defined for 'curl' and 'wget' than this
 * implementation can support. Some which we should support soon are:
 *
 *     --connect-timeout SECONDS  Maximum time allowed for connection
 *     --ftp-pasv      Use PASV/EPSV instead of PORT
 * -d, --data DATA     HTTP POST data
 * -G, --get           Send the -d data with a HTTP GET
 *     --data-ascii DATA  HTTP POST ASCII data
 *     --data-binary DATA  HTTP POST binary data
 *     --data-urlencode DATA  HTTP POST data url encoded
 * -n, --netrc         Must read .netrc for user name and password
 * -I, --head          Show document info only
 */