/*	HyperText Tranfer Protocol	- Client implementation		HTTP.c
**	==========================
** Modified:
** 27 Jan 1994  PDM  Added Ari Luotonen's Fix for Reload when using proxy
**                   servers.
*/

#include "HTUtils.h"
#include "tcp.h"

#include "HTTP.h"

#define HTTP_VERSION	"HTTP/1.0"

#define HTTP_PORT   80
#define HTTPS_PORT  443
#define SNEWS_PORT  563

#define HTTP_NETREAD   NETREAD
#define HTTP_NETWRITE  NETWRITE

#define INIT_LINE_SIZE		1024	/* Start with line buffer this big */
#define LINE_EXTEND_THRESH	256	/* Minimum read size */
#define VERSION_LENGTH 		20	/* for returned protocol version */

#include "HTParse.h"
#include "HTTCP.h"
#include "HTFormat.h"
#include "HTFile.h"
#include <ctype.h>
#include "HTAlert.h"
#include "HTMIME.h"
#include "HTML.h"
#include "HTInit.h"
#include "HTAABrow.h"

#include "LYLeaks.h"

#ifdef NO_BCOPY
#define bcopy(s, d, n) memcpy((d), (s), (n))
#endif /* NO_BCOPY */

/* #define TRACE 1 */

struct _HTStream 
{
  HTStreamClass * isa;
};

extern char * HTAppName;	/* Application name: please supply */
extern char * HTAppVersion;	/* Application version: please supply */
extern char * personal_mail_address;	/* User's name/email address */
extern char * LYUserAgent;	/* Lynx User-Agent string */
extern BOOL LYNoRefererHeader;	/* Never send Referer header? */
extern BOOL LYNoFromHeader;	/* Never send From header? */

extern BOOL using_proxy;	/* Are we using an HTTP gateway? */
PUBLIC BOOL reloading = FALSE;	/* Reloading => send no-cache pragma to proxy */
PUBLIC BOOL permanent_redirection = FALSE;  /* Got 301 status? */

extern char LYUserSpecifiedURL; /* Is the URL a goto? */

extern BOOL keep_mime_headers;   /* Include mime headers and force source dump */
extern BOOL no_url_redirection;  /* Don't follow Location: URL for */
extern char *http_error_file;    /* Store HTTP status code in this file */
extern BOOL traversal;		 /* TRUE if we are doing a traversal */

PUBLIC BOOL HTUseRawLatin = FALSE;

/*		Load Document from HTTP Server			HTLoadHTTP()
**		==============================
**
**	Given a hypertext address, this routine loads a document.
**
**
** On entry,
**	arg	is the hypertext reference of the article to be loaded.
**
** On exit,
**	returns	>=0	If no error, a good socket number
**		<0	Error.
**
**	The socket must be closed by the caller after the document has been
**	read.
**
*/
PUBLIC int HTLoadHTTP ARGS4 (
	CONST char *, 		arg,
	HTParentAnchor *,	anAnchor,
	HTFormat,		format_out,
	HTStream*,		sink)
{
  int s;			/* Socket number for returned data */
  char *url = (char *)arg;	/* The URL which get_physical() returned */
  char *command=NULL;		/* The whole command */
  char *eol;			/* End of line if found */
  char *start_of_data;		/* Start of body of reply */
  int status;			/* tcp return */
  int bytes_already_read;
  char crlf[3];			/* A CR LF equivalent string */
  HTStream *target;		/* Unconverted data */
  HTFormat format_in;		/* Format arriving in the message */
  int do_head = 0;		/* Whether or not we should do a head */
  int do_post = 0;		/* ARE WE posting ? */
  char *METHOD;

  BOOL had_header;		/* Have we had at least one header? */
  char *line_buffer;
  char *line_kept_clean;
  BOOL extensions;		/* Assume good HTTP server */
  int compressed;
  char line[256];

  int length, doing_redirect, rv;
  int already_retrying = 0;

  HTUseRawLatin = FALSE;

  if(anAnchor->isHEAD)
      do_head = TRUE;
  else if(anAnchor->post_data)
      do_post = TRUE;

  if (!url)
    {
      status = -3;
      _HTProgress ("Bad request.");
      goto done;
    }
  if (!*url) 
    {
      status = -2;
      _HTProgress ("Bad request.");
      goto done;
    }

  sprintf(crlf, "%c%c", CR, LF);

  /* At this point, we're talking HTTP/1.0. */
  extensions = YES;

 try_again:
  /* All initializations are moved down here from up above,
     so we can start over here... */
  eol = 0;
  bytes_already_read = 0;
  had_header = NO;
  length = 0;
  doing_redirect = 0;
  permanent_redirection = FALSE;
  compressed = 0;
  target = NULL;
  line_buffer = NULL;
  line_kept_clean = NULL;

  if(!strncmp(url, "https", 5))
    {
      HTAlert("This client does not contain support for HTTPS URLs.");
      status = HT_NO_DATA;
      goto done;
    }
  status = HTDoConnect (arg, "HTTP", HTTP_PORT, &s);
  if (status == HT_INTERRUPTED)
    {
      /* Interrupt cleanly. */
      if (TRACE)
        fprintf (stderr,
                 "HTTP: Interrupted on connect; recovering cleanly.\n");
      _HTProgress ("Connection interrupted.");
      /* status already == HT_INTERRUPTED */
      goto done;
    }
  if (status < 0) 
    {
      if (TRACE) 
        fprintf(stderr, 
            "HTTP: Unable to connect to remote host for `%s' (errno = %d).\n",
	    url, SOCKET_ERRNO);
      HTAlert("Unable to connect to remote host.");
      status = HT_NO_DATA;
      goto done;
    }

  /*	Ask that node for the document,
  **	omitting the host name & anchor
  */
  {
    char * p1 = (HTParse(url, "", PARSE_PATH|PARSE_PUNCTUATION));

    if (do_post) {
      METHOD = "POST";
      StrAllocCopy(command, "POST ");
    } else if(do_head) {
      METHOD = "HEAD";
      StrAllocCopy(command, "HEAD ");
    } else {
      METHOD = "GET";
      StrAllocCopy(command, "GET ");
    }

    /* if we are using a proxy gateway don't copy in the first slash
     * of say: /gopher://a;lkdjfl;ajdf;lkj/;aldk/adflj
     * so that just gopher://.... is sent.
     */
    if (using_proxy)
        StrAllocCat(command, p1+1);
    else
        StrAllocCat(command, p1);
    FREE(p1);
  }
  if (extensions) 
    {
      StrAllocCat(command, " ");
      StrAllocCat(command, HTTP_VERSION);
    }

  StrAllocCat(command, crlf);	/* CR LF, as in rfc 977 */

  if (extensions) 
    {
      int n, i;
      char * host = NULL;
      extern char * language; /* Lynx's preferred language - FM */
      extern char * pref_charset; /* Lynx's preferred character set - MM */

      if ((host = HTParse(anAnchor->address, "", PARSE_HOST)) != NULL) {
	  sprintf(line, "Host: %s%c%c", host, CR,LF);
	  StrAllocCat(command, line);
	  FREE(host);
      }

      if (!HTPresentations) HTFormatInit();
      n = HTList_count(HTPresentations);

      for(i=0; i<n; i++) 
        {
          HTPresentation * pres = HTList_objectAt(HTPresentations, i);
          if (pres->rep_out == WWW_PRESENT)
            {
	      if(pres->rep == WWW_SOURCE) 
		  sprintf(line, "Accept: */*%c%c", CR, LF);
  	      else
                  sprintf(line, "Accept: %s%c%c", 
					HTAtom_name(pres->rep), CR, LF);
              StrAllocCat(command, line);
            }
        }

      if (language && *language) {
          sprintf(line, "Accept-Language: %s%c%c", language, CR, LF);
	  StrAllocCat(command, line);
      }

      if (pref_charset && *pref_charset) {
	  StrAllocCat(command, "Accept-Charset: ");
	  strcpy(line, pref_charset);
	  if (line[strlen(line)-1] == ',')
	      line[strlen(line)-1] = '\0';
	  for (i = 0; line[i]; i++)
	      line[i] = TOLOWER(line[i]);
	  if (strstr(line, "iso-8859-1") == NULL)
	      strcat(line, ", iso-8859-1;q=0.001");
	  if (strstr(line, "us-ascii") == NULL)
	      strcat(line, ", us-ascii;q=0.001");
	  StrAllocCat(command, line);
	  sprintf(line, "%c%c", CR, LF);
	  StrAllocCat(command, line);
      }

      /*
       * When reloading give no-cache pragma to proxy server to make
       * it refresh its cache. -- Ari L. <luotonen@dxcern.cern.ch>
       *
       * Also send it as a Cache-Control header for HTTP/1.1. - FM
       */
      if (reloading) {
          sprintf(line, "Pragma: no-cache%c%c", CR, LF);
          StrAllocCat(command, line);
          sprintf(line, "Cache-Control: no-cache%c%c", CR, LF);
          StrAllocCat(command, line);
      }
      reloading = FALSE;           /* Now turn it off again if on */

      if (LYUserAgent && *LYUserAgent) {
          sprintf(line, "User-Agent: %s%c%c", LYUserAgent, CR, LF);
      } else {
          sprintf(line, "User-Agent: %s/%s  libwww-FM/%s%c%c",
                  HTAppName ? HTAppName : "unknown",
		  HTAppVersion ? HTAppVersion : "0.0",
		  HTLibraryVersion, CR, LF);
      }
      StrAllocCat(command, line);

      if(personal_mail_address && !LYNoFromHeader) {
          sprintf(line, "From: %s%c%c", personal_mail_address, CR,LF);
          StrAllocCat(command, line);
      }

      if(!(LYUserSpecifiedURL || LYNoRefererHeader)) {
          StrAllocCat(command, "Referer: ");
          StrAllocCat(command, (char *)HTLoadedDocumentURL());
          sprintf(line, "%c%c", CR, LF);
          StrAllocCat(command, line);
      }

      {
        char *docname;
        char *hostname;
        char *colon;
        int portnumber;
        char *auth;

        docname = HTParse(url, "", PARSE_PATH);
        hostname = HTParse(url, "", PARSE_HOST);
        if (hostname &&
            NULL != (colon = strchr(hostname, ':'))) 
          {
            *(colon++) = '\0';	/* Chop off port number */
            portnumber = atoi(colon);
          }
        else if(!strncmp(url, "https", 5))
	  {
	    portnumber = HTTPS_PORT;
	  }
        else 
	  {
	    portnumber = HTTP_PORT;
	  }

        if (NULL!=(auth=HTAA_composeAuth(hostname, portnumber, docname))) 
          {
            sprintf(line, "%s%c%c", auth, CR, LF);
            StrAllocCat(command, line);
          }
        if (TRACE) 
          {
            if (auth)
              fprintf(stderr, "HTTP: Sending authorization: %s\n", auth);
            else
              fprintf(stderr, "HTTP: Not sending authorization (yet)\n");
          }
        FREE(hostname);
        FREE(docname);
      }
    }

  if (do_post)
    {
      if (TRACE)
        fprintf (stderr, "HTTP: Doing post, content-type '%s'\n",
                 anAnchor->post_content_type);
      sprintf (line, "Content-type: %s%c%c",
               anAnchor->post_content_type ? anAnchor->post_content_type 
							: "lose", CR, LF);
      StrAllocCat(command, line);
      {
        int content_length;
        if (!anAnchor->post_data)
          content_length = 4; /* 4 == "lose" :-) */
        else
          content_length = strlen (anAnchor->post_data);
        sprintf (line, "Content-length: %d%c%c",
                 content_length, CR, LF);
        StrAllocCat(command, line);
      }

      StrAllocCat(command, crlf);	/* Blank line means "end" */

      StrAllocCat(command, anAnchor->post_data);
    }

  StrAllocCat(command, crlf);	/* Blank line means "end" */

  if (TRACE)
    fprintf (stderr, "Writing:\n%s----------------------------------\n",
             command);

  _HTProgress ("Sending HTTP request.");

  status = HTTP_NETWRITE(s, command, (int)strlen(command));
  FREE(command);
  if (status <= 0) 
    {
      if (status == 0)
        {
          if (TRACE)
            fprintf (stderr, "HTTP: Got status 0 in initial write\n");
          /* Do nothing. */
        }
      else if 
        ((SOCKET_ERRNO == ENOTCONN || SOCKET_ERRNO == ECONNRESET || SOCKET_ERRNO == EPIPE) &&
         !already_retrying &&
         /* Don't retry if we're posting. */ !do_post)
          {
            /* Arrrrgh, HTTP 0/1 compability problem, maybe. */
            if (TRACE)
              fprintf 
                (stderr, 
                 "HTTP: BONZO ON WRITE Trying again with HTTP0 request.\n");
            _HTProgress ("Retrying as HTTP0 request.");
            (void)NETCLOSE(s);
            extensions = NO;
            already_retrying = 1;
            goto try_again;
          }
      else
        {
          if (TRACE)
            fprintf (stderr,
	   "HTTP: Hit unexpected network WRITE error; aborting connection.\n");
          (void)NETCLOSE(s);
          status = -1;
          HTAlert("Unexpected network write error; connection aborted.");
          goto done;
        }
    }

  if (TRACE)
    fprintf (stderr, "HTTP: WRITE delivered OK\n");
  _HTProgress ("HTTP request sent; waiting for response.");

  /*	Read the first line of the response
   **	-----------------------------------
   */
  
  {
    /* Get numeric status etc */
    BOOL end_of_file = NO;
    int buffer_length = INIT_LINE_SIZE;

    line_buffer = (char *) malloc(buffer_length * sizeof(char));

    do 
      {	/* Loop to read in the first line */
        /* Extend line buffer if necessary for those crazy WAIS URLs ;-) */
        if (buffer_length - length < LINE_EXTEND_THRESH) 
          {
            buffer_length = buffer_length + buffer_length;
            line_buffer = 
              (char *) realloc(line_buffer, buffer_length * sizeof(char));
          }
        if (TRACE)
          fprintf (stderr, "HTTP: Trying to read %d\n",
                   buffer_length - length - 1);
        status = HTTP_NETREAD(s, line_buffer + length,
                              buffer_length - length - 1);
        if (TRACE)
          fprintf (stderr, "HTTP: Read %d\n", status);
        if (status <= 0) 
          {
            /* Retry if we get nothing back too; 
               bomb out if we get nothing twice. */
            if (status == HT_INTERRUPTED)
              {
                if (TRACE)
                  fprintf (stderr, "HTTP: Interrupted initial read.\n");
                _HTProgress ("Connection interrupted.");
                status = HT_INTERRUPTED;
                goto clean_up;
              }
            else if 
              (status < 0 &&
               (SOCKET_ERRNO == ENOTCONN || SOCKET_ERRNO == ECONNRESET || 
		     SOCKET_ERRNO == EPIPE) && !already_retrying && !do_post)
              {
                /* Arrrrgh, HTTP 0/1 compability problem, maybe. */
                if (TRACE)
                  fprintf (stderr, "HTTP: BONZO Trying again with HTTP0 request.\n");
                (void)NETCLOSE(s);
                if (line_buffer) 
                  FREE(line_buffer);
                if (line_kept_clean) 
                  FREE(line_kept_clean);

                extensions = NO;
                already_retrying = 1;
                _HTProgress ("Retrying as HTTP0 request.");
                goto try_again;
              }
            else
              {
                if (TRACE)
                  fprintf (stderr,
  "HTTP: Hit unexpected network read error; aborting connection; status %d.\n",
			   status);
                HTAlert("Unexpected network read error; connection aborted.");
                (void)NETCLOSE(s);
                status = -1;
                goto clean_up;
              }
          }

        bytes_already_read += status;
        {
          char line[256];
          sprintf (line, "Read %d bytes of data.", bytes_already_read);
          HTProgress (line);
        }

#ifdef UCX  /* UCX returns -1 on EOF */
        if (status == 0 || status == -1) 
#else
        if (status == 0)
#endif
          {
            end_of_file = YES;
            break;
          }
        line_buffer[length+status] = 0;

        if (line_buffer)
          {
            if (line_kept_clean)
              FREE(line_kept_clean);
            line_kept_clean = (char *)malloc (buffer_length * sizeof (char));
            bcopy (line_buffer, line_kept_clean, buffer_length);
          }

        eol = strchr(line_buffer + length, LF);
        /* Do we *really* want to do this? */
        if (eol && eol != line_buffer && *(eol-1) == CR) 
          *(eol-1) = ' '; 

        length = length + status;

        /* Do we really want to do *this*? */
        if (eol) 
          *eol = 0;		/* Terminate the line */
      }
    /* All we need is the first line of the response.  If it's a HTTP/1.0
       response, then the first line will be absurdly short and therefore
       we can safely gate the number of bytes read through this code
       (as opposed to below) to ~1000. */
    /* Well, let's try 100. */
    while (!eol && !end_of_file && bytes_already_read < 100);
  } /* Scope of loop variables */


  /*	We now have a terminated unfolded line. Parse it.
   **	-------------------------------------------------
   */
  if (TRACE)
    fprintf(stderr, "HTTP: Rx: %s\n", line_buffer);

/* Kludge to work with old buggy servers and the VMS Help gateway.
** They can't handle the third word, so we try again without it.
*/
  if (extensions &&       /* Old buggy server or Help gateway? */
      (0==strncmp(line_buffer,"<TITLE>Bad File Request</TITLE>",31) ||
       0==strncmp(line_buffer,"Address should begin with",25) ||
       0==strncmp(line_buffer,"<TITLE>Help ",12) ||
       0==strcmp(line_buffer,
       		 "Document address invalid or access not authorised"))) {
      if (line_buffer)
	  FREE(line_buffer);
      if (line_kept_clean) 
          FREE(line_kept_clean);
      extensions = NO;
      already_retrying = 1;
      if (TRACE) fprintf(stderr,
			 "HTTP: close socket %d to retry with HTTP0\n", s);
      (void)NETCLOSE(s);
      /* print a progress message */
      _HTProgress ("Retrying as HTTP0 request.");
      goto try_again;
  }


  {
    int fields;
    char server_version[VERSION_LENGTH+1];
    int server_status;

    server_version[0] = 0;

    fields = sscanf(line_buffer, "%20s %d",
                    server_version,
                    &server_status);

    if (TRACE)
      fprintf (stderr, "HTTP: Scanned %d fields from line_buffer\n", fields);

    if (http_error_file) {     /* Make the status code externally available */
	FILE *error_file;
#ifdef SERVER_STATUS_ONLY
	error_file = fopen(http_error_file, "w");
	if (error_file) {		/* Managed to open the file */
	    fprintf(error_file, "error=%d\n", server_status);
#else
	error_file = fopen(http_error_file, "a");
	if (error_file) {		/* Managed to open the file */
	    fprintf(error_file, "   URL=%s (%s)\n", url, METHOD);
	    fprintf(error_file, "STATUS=%s\n", line_buffer);
#endif /* SERVER_STATUS_ONLY */
	    fclose(error_file);
	}
    }

    /* Rule out HTTP/1.0 reply as best we can. */
    if (fields < 2 || !server_version[0] || server_version[0] != 'H' ||
        server_version[1] != 'T' || server_version[2] != 'T' ||
        server_version[3] != 'P' || server_version[4] != '/' ||
        server_version[6] != '.') 
      {			/* HTTP0 reply */
        HTAtom * encoding;

        if (TRACE)
          fprintf (stderr, "--- Talking HTTP0.\n");

        format_in = HTFileFormat(url, &encoding);
	/* treat all plain text as HTML.
         * this sucks but its the only solution without
         * looking at content.
         */
        if(!strncmp(HTAtom_name(format_in), "text/plain",10)) {
            if(TRACE)
                fprintf(stderr,
                           "HTTP: format_in being changed to text/HTML\n");
            format_in = WWW_HTML;
        }

        start_of_data = line_kept_clean;
      } 
    else 
      {
        /* Decode full HTTP response */
        format_in = HTAtom_for("www/mime");
        /* We set start_of_data to "" when !eol here because there
           will be a put_block done below; we do *not* use the value
           of start_of_data (as a pointer) in the computation of
           length or anything else in this situation. */
        start_of_data = eol ? eol + 1 : "";
        length = eol ? length - (start_of_data - line_buffer) : 0;

        if (TRACE)
          fprintf (stderr, "--- Talking HTTP1.\n");

        switch (server_status / 100) 
          {
          case 3:		/* Various forms of redirection */
	    if (no_url_redirection || do_head || keep_mime_headers) {
	        /* We want to see Location: */
	        break;
	    }
            /*
	     * We do not load the file, but read the headers for
	     * the "Location: ", check out that redirecting_url
	     * and if it's acceptible (e.g., not a telnet URL
	     * when we have that disabled), initiate a new fetch.
	     * If that's another redirecting_url, we'll repeat the
	     * checks, and fetch initiations if acceptible, until
	     * we reach the actual URL.  If the status was 301,
	     * indicating that the relocation is permanent, we set
	     * the permanent_redirection flag to make it permanent
	     * for the current anchor tree (i.e., will persist until
	     * the tree is freed or the client exits).- FM
	     */
	    {
	      char *cp;
	      doing_redirect = 1;
	      if (server_status == 301) {
	          permanent_redirection = FALSE;
	          if(TRACE)
	              fprintf(stderr,
		      	      "[HTLoadHTTP] Got Permanent Redirect code\n");
	      } else {
	          if(TRACE)
	              fprintf(stderr,"[HTLoadHTTP] Got Redirect code\n");
	      }

	      /*
	       * Get the rest of the headers and data, if any,
	       * and then close the connection. - FM
	       */
	      while ((status = HTTP_NETREAD(s, line_buffer,
	      				    INIT_LINE_SIZE)) > 0) {
	          line_buffer[status] = '\0';
		  StrAllocCat(line_kept_clean, line_buffer);
	      }
	      (void)NETCLOSE(s);
              if (status == HT_INTERRUPTED) {
		  /*
		   * Impatient user. - FM
		   */
                  if (TRACE)
                      fprintf (stderr, "HTTP: Interrupted followup read.\n");
                  _HTProgress ("Connection interrupted.");
                  status = HT_INTERRUPTED;
                  goto clean_up;
              }

	      /*
	       * Look for the "Location: " in the headers. - FM
	       */
	      cp = line_kept_clean;
	      while (*cp) {
	        if(*cp != 'l' && *cp != 'L')
		    cp++;
	        else if (!strncasecomp(cp, "Location: ", 10)) {
	            extern char *redirecting_url;
	            char *cp1=NULL, *cp2=NULL;
	            cp += 10;
		    if((cp1=strchr(cp, LF)) != NULL) {
		        *cp1 = '\0';
		        if((cp2=strchr(cp, CR)) != NULL)
		            *cp2 = '\0';
			/*
			 * Load the new URL into redirecting_url,
			 * which will serve as a flag for seeking
			 * the document at that location. - FM
			 */
		        StrAllocCopy(redirecting_url, cp);
                        if (TRACE)
                            fprintf(stderr,
                                "[HTLoadHTTP] Picked up location '%s'\n", cp);
		        if(cp2)
		            *cp2 = CR;
		        *cp1 = LF;
		        status = HT_REDIRECTING;
		        goto clean_up;
		    }
	            break;
	        }
	        else {
	           cp++;
	        }
	      }
	      /*
	       * If we get to here, we didn't find the "Location: "
	       * header.  Better luck next time,  but we'll give up
	       * on this request. - FM
	       */
	      permanent_redirection = FALSE;
              if (TRACE)
                fprintf (stderr,
		    "[HTLoadHTTP] Failed to pickup location.\n");
              status = HT_NO_DATA;
              goto clean_up;
	   }

          case 4:		/* "I think I goofed" */
            switch (server_status) 
              {
              case 403:
                /* 403 is "forbidden"; display returned text. */
                /* format_in = HTAtom_for("text/html"); */
		if (traversal) {
		    (void)NETCLOSE(s);
		    status = -1;
		    goto clean_up;
		}
                break;

              case 401:
                /* length -= start_of_data - text_buffer; */
                if (HTAA_shouldRetryWithAuth(start_of_data, length, s)) 
                  {
                    extern BOOLEAN dump_output_immediately;
 		    extern char *authentication_info[2];

                    (void)NETCLOSE(s);
                    if(dump_output_immediately && !authentication_info[0]) {
                      fprintf(stderr,
		      		"HTTP: Access authorization required.\n");
                      fprintf(stderr,
		      		"       Use the -auth=id:pw parameter.\n");
                      status = HT_NO_DATA;
                      goto clean_up;
                    }

                    if (TRACE) 
                      fprintf(stderr, "%s %d %s\n",
                              "HTTP: close socket", s,
                              "to retry with Access Authorization");

                    _HTProgress (
		    	"Retrying with access authorization information.");
                    goto try_again;
                    break;
                  }
                else 
                  {
		    HTAlert(
	"Can't retry with authorization!  Contact the server's WebMaster.");
		    (void)NETCLOSE(s);
                    status = -1;
                    goto clean_up;
                  }

              default:
		if (traversal) {
		    (void)NETCLOSE(s);
		    status = -1;
		    goto clean_up;
		}
                break;
#if 0
                char *p1 = HTParse(url, "", PARSE_HOST);
                char *message;

                message = (char*)malloc(strlen(text_buffer) +
                                        strlen(p1) + 100);
                sprintf(message,
                        "HTTP server at %s replies:\n%s\n\n%s\n",
                        p1, text_buffer,
                        ((server_status == 401) 
                         ? "Access Authorization package giving up.\n"
                         : ""));
                FREE(message);
                FREE(p1);
#endif
#if 0
                _HTProgress ("Could not load requested document.");
                status = -1;
                goto clean_up;
#endif
              } /* case 4 switch */
            break;

          case 5:		/* I think you goofed */
            break;

          case 2:		/* Good: Got MIME object */
	    if(server_status == 204) {
	        status = HT_NO_DATA;
	        goto done;
	    }
            break;

          default:		/* bad number */
            HTAlert("Unknown status reply from server!");
	    if (traversal) {
		(void)NETCLOSE(s);
		status = -1;
		goto clean_up;
	    }
            break;
          } /* Switch on server_status/100 */

      }	/* Full HTTP reply */
  } /* scope of fields */

  /* Set up the stream stack to handle the body of the message */
  if (do_head || keep_mime_headers) {
      /* It was a HEAD request, or we want the headers and source */
      start_of_data = line_kept_clean;
      length = strlen(start_of_data);
      format_in = HTAtom_for("text/plain");
  }

  target = HTStreamStack(format_in,
                         format_out,
                         sink, anAnchor);

  if (!target || target == NULL) 
    {
      char buffer[1024];	/* @@@@@@@@ */

      (void)NETCLOSE(s);
      sprintf(buffer, "Sorry, no known way of converting %s to %s.",
              HTAtom_name(format_in), HTAtom_name(format_out));
      _HTProgress (buffer);
      status = -1;
      goto clean_up;
    }

  /* Recycle the first chunk of data, in all cases. */
  (*target->isa->put_block)(target, start_of_data, length);

  /* Go pull the bulk of the data down. */
  if (anAnchor->charset &&
      (!strcasecomp(anAnchor->charset, "iso-8859-2") ||
       !strcasecomp(anAnchor->charset, "koi8-r")))
      HTUseRawLatin = TRUE;
  rv = HTCopy(s, NULL, target);
  HTUseRawLatin = FALSE;

  if (rv == -1)
    {
      /* Intentional interrupt before data were received, not an error */
      /* (*target->isa->_abort)(target, NULL); */ /* already done in HTCopy */
      status = HT_INTERRUPTED;
      (void)NETCLOSE(s);
      goto clean_up;
    }
  if (rv == -2 && !already_retrying && !do_post)
    { 
      /* Aw hell, a REAL error, maybe cuz it's a dumb HTTP0 server */
      if (TRACE)
        fprintf (stderr, "HTTP: Trying again with HTTP0 request.\n");
      /* May as well consider it an interrupt -- right? */
      (*target->isa->_abort)(target, NULL);
      (void)NETCLOSE(s);
      if (line_buffer) 
        FREE(line_buffer);
      if (line_kept_clean) 
        FREE(line_kept_clean);

      extensions = NO;
      already_retrying = 1;
      _HTProgress ("Retrying as HTTP0 request.");
      goto try_again;
    }

  /* 
   * Close socket if partial transmission (was freed on abort)
   * Free if complete transmission (socket was closed before return)
   */
  if (rv == HT_INTERRUPTED)
    {
      (void)NETCLOSE(s);
    }
  else
      (*target->isa->_free)(target);

  if (doing_redirect)
    /*
     * We already jumped over all this if the "case 3:" code worked
     * above, but we'll check here as a backup in case it fails. - FM
     */
    {
      /* Lou's old comment:  - FM */
      /* OK, now we've got the redirection URL temporarily stored
         in external variable redirecting_url, exported from HTMIME.c,
         since there's no straightforward way to do this in the library
         currently.  Do the right thing. */
      status = HT_REDIRECTING;
    }
  else
    {
      /* If any data were received, treat as a complete transmission */
      status = HT_LOADED;
    }

  /*	Clean up
   */
clean_up:
  if (line_buffer) 
    FREE(line_buffer);
  if (line_kept_clean) 
    FREE(line_kept_clean);

done:
  /* Clear out on exit, just in case. */
  do_head = 0;
  do_post = 0;
  return status;
}


/*	Protocol descriptor
*/

#ifdef GLOBALDEF_IS_MACRO
#define _HTTP_C_GLOBALDEF_1_INIT { "http", HTLoadHTTP, 0}
GLOBALDEF (HTProtocol,HTTP,_HTTP_C_GLOBALDEF_1_INIT);
#define _HTTP_C_GLOBALDEF_2_INIT { "https", HTLoadHTTP, 0}
GLOBALDEF (HTProtocol,HTTPS,_HTTP_C_GLOBALDEF_2_INIT);
#else
GLOBALDEF PUBLIC HTProtocol HTTP = { "http", HTLoadHTTP, 0 };
GLOBALDEF PUBLIC HTProtocol HTTPS = { "https", HTLoadHTTP, 0 };
#endif /* GLOBALDEF_IS_MACRO */
