Retrieving files or directories by FTP

	This is an agent conforming to the W3A specification. It
	exports the functions [[initFTP]], [[openFTP]], [[readFTP]],
	[[writeFTP]], [[infoFTP]], and [[deleteFTP]].

	The URL is of the form ftp://host/path or file://host/path. A
	special case is when the host is `localhost'. In that case the
	file is first searched for on the local file system. The host
	part may also include a username and password, otherwise
	anonymous FTP is used. The format for a username and password
	is ftp://user:passw@host/rest. The password may be omitted if
	it is not needed.

	It is in general not possible to know beforehand whether the
	path is the path of a file or a directory. A trailing [[/]]
	could be an indication, but if it is not present, the agent
	first tries to retrieve the path as a file, if that fails it
	tries to CWD to it and asks for the list of files.

	It is also not possible to know whether the indicated file is
	binary. A set of heuristics is tried (just as in the Gopher
	agent, in fact).

	The output is described in a [[W3ADocumentInfo]] structure
	that indicates the probable format. If the output is a
	directory listing, the type is text/html. The directory is
	then listed with just the file names one per line, with
	hyperlinks inserted. This could be made much more fancy, with
	code to parse and format `ls -l' directory listings.

	The current implementation doesn't deal correctly with
	non-blocking I/O. [[readFTP]] is correct, but the [[openFTP]]
	call could still block, waiting for the connection to be
	established.

	The FTP protocol is defined in RFC959.

	Bert Bos [[<bert@let.rug.nl>]], 1994

<<*>>=
static char copyright[] = "Copyright 1994 NBBI, Den Haag";
#define NO_X11
#include <config.h>
#include <time.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <netdb.h>
#include <signal.h>
#include <fcntl.h>
#include <str.h>
#include <url.h>
#include <w3a.h>

#define min(a, b) ((a) < (b) ? (a) : (b))

#define MAXSTR 256

static struct {					/* Describes one connection */
    char mime_type[MAXSTR];
    int len;					/* Of mime_type */
    Bool type_known;				/* mime_type valid? */
    pid_t child;
} *conn_info[FD_SETSIZE];

static Strip FILE_T = NULL;
static Strip FTP_T = NULL;
static Strip LOCALHOST_T = NULL;
@

<<*>>=
static char *image(char ftype, char *fname)
{
    Bool binary;
    char *type, *encoding;

    if (ftype == 'd') return "&folder;";
    else if (! URL_infer_type(fname, FALSE, &type, &encoding, &binary))
	return "&unknown.document;";
    else if (eq(type, "application/x-tar")) return "&archive;";
    else if (eq(type, "application/binary")) return "&binary.document;";
    else if (n_eq(type, "image/", 6)) return "&image;";
    else if (n_eq(type, "text/", 5)) return "&text.document;";
    else if (eq(encoding, "x-compress")) return "&compressed.document;";
    else if (eq(encoding, "x-gzip")) return "&compressed.document;";
    else if (eq(encoding, "x-uuencode")) return "&uuencoded.document;";
    else return "&document;";
}
@

	[[ftp_cmd]]: Send an FTP command and get response

	The command is written to file (socket) [[f]]. The command is
	a [[printf]]-style string, with a variable-length argument
	list. It should not include a CR/LF pair at the end; they will
	be added by the [[ftp_cmd]] function automatically. The first
	response line is returned in [[buf]], which is assumed to be
	[[BUFSIZ]] characters long. The status code that is returned
	is the first character of [[buf]], which should be '1', '2',
	'3', '4' or '5'.
        
<<*>>=
static char ftp_cmd(FILE *f, char *buf, char *format, ...)
{
    va_list args;
    char s[BUFSIZ], c;
    int n;

    if (format) {                               /* Command to send? */
        va_start(args, format);
        fflush(f);
        vfprintf(f, format, args); putc('\015', f); putc('\012', f);
        fflush(f);
        va_end(args);
    }
    if (!fgets(buf, BUFSIZ, f)) return '5';     /* Read response into buf */
    if (buf[3] == '-')                          /* Read continuation lines */
        while (fgets(s, sizeof(s), f) && (s[3] != ' ' || !n_eq(s, buf, 3))) ;
    /* debug("-> %s", buf); */
    return buf[0];                              /* Return status code */
}
@

	[[open_data_socket]] uses the PASV command to ask the remote
	host for a data connection. The FTP server returns the IP
	address and the port number as 6 decimal numbers: 4 for the
	internet address and 2 for the port.

<<*>>=
static FILE *open_data_socket(FILE *f)
{
    char s[BUFSIZ], dataaddr[100], dataport[100];
    int i, h1, h2, h3, h4, p1, p2;
    int datasock;

    if (ftp_cmd(f, s, "PASV") != '2') return NULL;
    for (i = 0; s[i]; i++) if (!isdigit(s[i])) s[i] = ' ';
    if (sscanf(s+4, "%d%d%d%d%d%d",&h1,&h2,&h3,&h4,&p1,&p2) != 6) return NULL;
    sprintf(dataaddr, "%d.%d.%d.%d", h1, h2, h3, h4);
    sprintf(dataport, "%d", 256 * p1 + p2);
    if ((datasock = connectTCP(dataaddr, dataport)) < 0) return NULL;
    return fdopen(datasock, "r+");
}
@

	The functions [[child_process]] does the actual work of
	connecting to the remote host, retrieving the file or
	directory, and translating the result to HTML in the case of a
	directory.

	Files will be closed automatically since the process will exit
	immediately.
	
<<*>>=
/* child_process -- retrieve file or directory and write it to pipe */
static void child_process(const char *url, int pipe_fd)
{
    static char spaces[] = "                        ";
    FILE *pipef, *f, *datasock;
    int n, day, status;
    long size;
    char *type, *encoding, *p, *prot, *dummy, *month, *time_or_year, *name;
    char buf[BUFSIZ], this_year[3], path[1024];
    Bool binary, is_year;
    URI uri;
    struct passwd *pwent;			/* Info about user */
    struct hostent *phe;			/* Info about localhost */
    char host[1024], e_mail[1024] = "WWWuser";	/* Name of local machine */
    char *username, *systemname, *password, *port;
    time_t tm;

    /* debug("child_process writing to %d\n", pipe_fd); */
    if (! (pipef = fdopen(pipe_fd, "w"))) {
	/* debug("FTP child exited with an error\n"); */
	exit(1);				/* Stop process */
    }
    if (! URL_parse(url, &uri)			/* Analyze URL */
        || (uri.tp!=URI_Rel && uri.scheme!=FTP_T && uri.scheme!=FILE_T)) {
	fprintf(pipef, "400\n");		/* Error status code */
	/* debug("FTP child exited with an error: incorrect URL\n"); */
	exit(1);				/* Stop process */
    }
    if (! URL_infer_type(strip2str(uri.path), FALSE, &type, &encoding,
			 &binary)) {
	/* debug("Type not known, defaulting to text/plain\n"); */
	binary = TRUE;
	type = newstring("text/plain");
	encoding = newstring("8bit");
    }
    dispose(encoding);				/* Not implemented yet */

    /* Try local file first */
    /* debug("scheme=%s host=%s path=\"%s\"\n",
	  strip2str(uri.scheme), strip2str(uri.host), strip2str(uri.path)); */
    assert(FILE_T == str2strip("file"));
    assert(LOCALHOST_T == str2strip("localhost"));
    if (uri.tp == URI_Rel || (uri.scheme==FILE_T && uri.host==LOCALHOST_T)){
	p = strip2str(uri.path);
	/* debug("Try local file \"%s\"\n", p); */
    	if ((f = fopen(p, "r"))) {
	    /* debug("FTP child: found local file %s, type=%s\n",
		  strip2str(uri.path), type); */
	    fprintf(pipef, "200 %s\n", type);	/* Status and MIME type */
	    /* debug("--> 200 %s\n", type);	/* Status and MIME type */
	    while ((n = fread(buf, 1, sizeof(buf), f)) > 0)
		fwrite(buf, 1, n, pipef);
	    /* debug("FTP child completed successfully\n"); */
	    exit(0);				/* Stop process */
	}
	/* debug("%s\n", strerror(errno)); */
    }

    /* Find out who we are */
    if ((pwent = getpwuid(getuid()))
	&& (gethostname(host, sizeof(host)) == 0)
	&& (phe = gethostbyname(host)))
	sprintf(e_mail, "%s@%s", pwent->pw_name, phe->h_name);

    username = uri.user ? strip2str(uri.user) : "anonymous";
    password = uri.user && uri.passw ? strip2str(uri.passw) : e_mail;
    port = uri.port ? strip2str(uri.port) : "21";
    
    /* Open a socket connection and log in */
    if (! (f = fdopen(connectTCP(strip2str(uri.host), port), "r+"))) {
	(void) fprintf(pipef, "400\n");		/* Error status code */
	/* debug("FTP child exited with an error: couldn't open connection\n");
	 */
	exit(1);				/* Stop process */
    }
    if (ftp_cmd(f, buf, NULL) != '2') {		/* Read welcome msg */
	(void) fprintf(pipef, "%s", buf);	/* Error status code */
	/* debug("FTP child error: %s\n", buf); */
	exit(1);				/* Stop process */
    }
    if ((status = ftp_cmd(f, buf, "USER %s", username)) == '3')
        if ((status = ftp_cmd(f, buf, "PASS %s", password)) == '3')
            status = ftp_cmd(f, buf, "ACCT %s", "noaccount");
    if (status != '2') {
	(void) fprintf(pipef, "%s", buf);	/* Error status code */
	/* debug("FTP child error: %s\n", buf); */
	exit(1);				/* Stop process */
    }
    if (ftp_cmd(f, buf, "SYST") != '2') {	/* Ask for system type */
	(void) fprintf(pipef, "%s", buf);	/* Error status code */
	/* debug("FTP child error: %s\n", buf); */
	exit(1);				/* Stop process */
    }
    systemname = newstring(tokenize(buf + 4, " \t\r\n", &p));

    if (ftp_cmd(f, buf, "TYPE %c", binary ? 'I' : 'A') != '2') {
	(void) fprintf(pipef, "%s", buf);	/* Error status code */
	/* debug("FTP child error: %s\n", buf); */
	exit(1);				/* Stop process */
    }

    /* Ask remote server to create passive socket, connect to it */
    if (! (datasock = open_data_socket(f))) {
	(void) fprintf(pipef, "400\n");		/* Error status code */
	/* debug("FTP child error: %s\n", buf); */
	exit(1);				/* Stop process */
    }

    /* Try to retrieve as file */
    if (ftp_cmd(f, buf, "RETR %s", strip2str(uri.path)) == '1') {
	fprintf(pipef, "200 %s\n", type);	/* Status and MIME type */
	/* debug("--> 200 %s\n", type);		/* Status and MIME type */
	while ((n = fread(buf, 1, sizeof(buf), datasock)) > 0)
	    fwrite(buf, 1, n, pipef);
	/* debug("FTP: child completed successfully\n"); */
	exit(0);				/* Stop process */
    }

    /* Try to interpret path as directory */
    if (ftp_cmd(f, buf, "CWD %s", strip2str(uri.path)) != '2') {
	(void) fprintf(pipef, "%s", buf);	/* Error status code */
	/* debug("FTP child error: %s\n", buf); */
	exit(1);				/* Stop process */
    }
    if (case_eq(systemname, "unix")) {		/* Unix => use long listing */
	if (ftp_cmd(f, buf, "LIST") != '1') {
	    (void) fprintf(pipef, "%s", buf);	/* Error status code */
	    /* debug("FTP: child error: %s\n", buf); */
	    exit(1);				/* Stop process */
	}
	tm = time(NULL);
	strftime(this_year, sizeof(this_year), "%y", localtime(&tm));
	strcpy(path, strip2str(uri.path));
	n = strlen(path);
	if (path[n-1] != '/') strcpy(path + n, "/");
	fprintf(pipef, "200 text/html\n");	/* Status and MIME type */
	fprintf(pipef, "<title>%s</title>\n", strip2str(uri.path));
	fprintf(pipef, "<h1>%s</h1>\n<pre>\n", strip2str(uri.path));
	fgets(buf, sizeof(buf), datasock);	/* Skip first line */
	while (fgets(buf, sizeof(buf), datasock)) {
	    /* debug("FTP child read: %s", buf); */
	    prot = tokenize(buf, " \r\n", &p);
	    dummy = tokenize(p, " \r\n", &p);	/* # Of links */
	    dummy = tokenize(p, " \r\n", &p);	/* Owner */
	    dummy = tokenize(p, " \r\n", &p);	/* Group */
	    size = atol(tokenize(p, " \r\n", &p));
	    month = tokenize(p, " \r\n", &p);
	    day = atoi(tokenize(p, " \r\n", &p));
	    time_or_year = tokenize(p, " \r\n", &p);
	    name = tokenize(p, "\r\n", &p);
	    is_year = strlen(time_or_year) == 4;
	    fprintf(pipef, "%s <a href=\"%s%s\">",
		    image(prot[0], name), path, name);
	    n = fprintf(pipef, "%s ", name);
	    if (n < sizeof(spaces)) fprintf(pipef, "</a>%s", spaces + n);
	    fprintf(pipef, "%02d-%s-%s %s %5ldK\n", day, month,
		    is_year ? time_or_year + 2 : this_year,
		    is_year ? "00:00" : time_or_year,
		    (size + 512)/1024);
	}
	fprintf(pipef, "</pre>\n");
	/* debug("FTP: child completed successfully\n"); */
	exit(0);				/* Stop process */
    }

    /* Not Unix, use plain list */
    if (ftp_cmd(f, buf, "NLST") != '1') {
	(void) fprintf(pipef, "%s", buf);	/* Error status code */
	/* debug("FTP child error: %s\n", buf); */
	exit(1);				/* Stop process */
    }
    fprintf(pipef, "200 %s\n", type);		/* Status and MIME type */
    while ((n = fread(buf, 1, sizeof(buf), datasock)) > 0)
	fwrite(buf, 1, n, pipef);
    /* debug("FTP: child completed successfully\n"); */
    exit(0);					/* Stop process */
}
@

	[[read_mime_type]] reads until the end of line and stores the
	line in [[mime_type]]. When end of line is reached, the
	function returns [[TRUE]] and sets [[type_known]] to [[TRUE]],
	otherwise the function returns [[FALSE]], indicating that the
	function has to be called again to read the rest of the data,
	once it is available.

<<*>>=
static Bool read_mime_type(int fd)
{
    char c = ' ';

    while (read(fd, &c, 1) > 0 && c != '\n')
	conn_info[fd]->mime_type[conn_info[fd]->len++] = c;
    if (c == '\n') {
	conn_info[fd]->mime_type[conn_info[fd]->len] = '\0';
	conn_info[fd]->type_known = TRUE;
	/* debug("FTP child: MIME type = %s\n", conn_info[fd]->mime_type); */
    }
    return conn_info[fd]->type_known;
}


EXPORT Bool initFTP(char ***protocols, int *nrprotocols)
{
    static char *proto[] = {"ftp", "file"};	/* Support 2 protocols */

    *protocols = proto;
    *nrprotocols = 2;

    FILE_T = str2strip("file");
    FTP_T = str2strip("ftp");
    LOCALHOST_T = str2strip("localhost");

    return TRUE;
}


EXPORT int openFTP(const char *url, int method, int flags, const char *referer)
{
    int fd[2];					/* Pipe */
    long c, tblsiz;
    pid_t child;

    if (pipe(fd) < 0) return -1;		/* Create pipe */
    switch ((child = fork())) {			/* Fork process */
    case -1: return -1;				/* Out of processes */
    case 0:
	/* debug("FTP: child is alive\n"); */
	/* Close all unneeded files */
	tblsiz = sysconf(_SC_OPEN_MAX);
	for (c = 3; c < tblsiz; c++) if (c != fd[1]) (void) close(c);
	child_process(url, fd[1]);
	/* NOTREACHED */
    }
    if (close(fd[1]) < 0) return -1;		/* I/O Error */
    if ((flags & O_NONBLOCK) && fcntl(fd[0], F_SETFL, O_NONBLOCK) < 0)
	return -1;				/* I/O error */
    new(conn_info[fd[0]]);
    conn_info[fd[0]]->len = 0;
    conn_info[fd[0]]->type_known = FALSE;
    conn_info[fd[0]]->child = child;
    return fd[0];
}


EXPORT Bool doneFTP(int fd)
{
    return TRUE;				/* No need to call doneFTP */
}


EXPORT int peekFTP(int fd)
{
#ifndef howmany
#define howmany(x,y)  (((x)+((y)-1))/(y))
#endif
#define NRBITS (sizeof(int) * 8)		/* 8 bits per byte */
    int mask[howmany(FD_SETSIZE, NRBITS)];
    struct timeval timeout;
    int n, i;

    timeout.tv_sec = 0;				/* Wait 0 seconds */
    timeout.tv_usec = 0;			/* .. and 0 milliseconds */
    for (i = 0; i < howmany(FD_SETSIZE, NRBITS); i++) mask[i] = 0;
    mask[fd/NRBITS] |= 1 << (fd % NRBITS);
    if ((n = select(fd + 1, mask, NULL, NULL, &timeout)) == -1) return -1;
    return n;					/* 0 or 1 */
}
@

	The [[readFTP]] function operates differently, depending on
	whether the thing to retrieve is a directory or a file. If it
	is a file, a number of bytes are read and copied to
	[[buffer]], without any translation.

	If the request was for a directory, the function reads one
	directory entry and generates an HTML expression for it. The
	generated HTML is stored in a

<<*>>=
EXPORT int readFTP(int fd, char *buffer, size_t nbytes)
{
    if (! conn_info[fd]->type_known && ! read_mime_type(fd)) return -1;
    return read(fd, buffer, nbytes);
}


EXPORT int writeFTP(int fd, const char *buf, size_t nbytes)
{
    errno = ENYI;				/* Not Yet Implemented */
    return -1;
}


EXPORT Bool closeFTP(int fd)
{
    (void) kill(conn_info[fd]->child, SIGHUP);
    (void) waitpid(conn_info[fd]->child, NULL, 0);
    dispose(conn_info[fd]);
    return close(fd) >= 0;
}


EXPORT Bool deleteFTP(const char *url)
{
    errno = ENYI;				/* Not Yet Implemented */
    return -1;
}


EXPORT Bool infoFTP(int fd, W3ADocumentInfo *buf)
{
    if (! conn_info[fd]->type_known && ! read_mime_type(fd)) return FALSE;
    dispose(buf->mime_type);
    dispose(buf->mime_params);
    buf->mime_type = newstring(conn_info[fd]->mime_type + 4);
    buf->mime_params = NULL;
    return TRUE;
}