/*
 *  broker.h -- Global definitions and data type for the broker.
 *  
 *  broker.h,v 1.108 1995/09/10 17:11:49 duane Exp
 *  ----------------------------------------------------------------------
 *  Copyright (c) 1994, 1995.  All rights reserved.
 *  
 *    The Harvest software was developed by the Internet Research Task
 *    Force Research Group on Resource Discovery (IRTF-RD):
 *  
 *          Mic Bowman of Transarc Corporation.
 *          Peter Danzig of the University of Southern California.
 *          Darren R. Hardy of the University of Colorado at Boulder.
 *          Udi Manber of the University of Arizona.
 *          Michael F. Schwartz of the University of Colorado at Boulder.
 *          Duane Wessels of the University of Colorado at Boulder.
 *  
 *    This copyright notice applies to software in the Harvest
 *    ``src/'' directory only.  Users should consult the individual
 *    copyright notices in the ``components/'' subdirectories for
 *    copyright information about other software bundled with the
 *    Harvest source code distribution.
 *  
 *  TERMS OF USE
 *    
 *    The Harvest software may be used and re-distributed without
 *    charge, provided that the software origin and research team are
 *    cited in any use of the system.  Most commonly this is
 *    accomplished by including a link to the Harvest Home Page
 *    (http://harvest.cs.colorado.edu/) from the query page of any
 *    Broker you deploy, as well as in the query result pages.  These
 *    links are generated automatically by the standard Broker
 *    software distribution.
 *    
 *    The Harvest software is provided ``as is'', without express or
 *    implied warranty, and with no support nor obligation to assist
 *    in its use, correction, modification or enhancement.  We assume
 *    no liability with respect to the infringement of copyrights,
 *    trade secrets, or any patents, and are not responsible for
 *    consequential damages.  Proper use of the Harvest software is
 *    entirely the responsibility of the user.
 *  
 *  DERIVATIVE WORKS
 *  
 *    Users may make derivative works from the Harvest software, subject 
 *    to the following constraints:
 *  
 *      - You must include the above copyright notice and these 
 *        accompanying paragraphs in all forms of derivative works, 
 *        and any documentation and other materials related to such 
 *        distribution and use acknowledge that the software was 
 *        developed at the above institutions.
 *  
 *      - You must notify IRTF-RD regarding your distribution of 
 *        the derivative work.
 *  
 *      - You must clearly notify users that your are distributing 
 *        a modified version and not the original Harvest software.
 *  
 *      - Any derivative product is also subject to these copyright 
 *        and use restrictions.
 *  
 *    Note that the Harvest software is NOT in the public domain.  We
 *    retain copyright, as specified above.
 *  
 *  HISTORY OF FREE SOFTWARE STATUS
 *  
 *    Originally we required sites to license the software in cases
 *    where they were going to build commercial products/services
 *    around Harvest.  In June 1995 we changed this policy.  We now
 *    allow people to use the core Harvest software (the code found in
 *    the Harvest ``src/'' directory) for free.  We made this change
 *    in the interest of encouraging the widest possible deployment of
 *    the technology.  The Harvest software is really a reference
 *    implementation of a set of protocols and formats, some of which
 *    we intend to standardize.  We encourage commercial
 *    re-implementations of code complying to this set of standards.  
 *  
 */
#ifndef _BROKER_H_
#define _BROKER_H_
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
#include <malloc.h>
#include <errno.h>
#include <time.h>
#include <ctype.h>
#include <memory.h>
#include <netdb.h>
#include <signal.h>
#include <limits.h>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/param.h>
#include <sys/time.h>
#include <varargs.h>
#include <sys/wait.h>
#include <sys/socket.h>
#ifdef _AIX
#include <sys/select.h>
#endif
#include <netinet/in.h>
#include "config.h"
#include "util.h"
#include "template.h"

/*
 *  MAX_EVENTS - max # of clients the broker will hold in its queue
 */
#define MAX_EVENTS      15

/*
 *  READ_QUERY_TIMEOUT - A hack to make sure we get the whole query.
 *  Sometimes TCP breaks up long query packets, so we wait for a
 *  timeout to occur before thinking we have the whole query.  This
 *  value is microseconds (500000 is 0.5 seconds).  This value
 *  can be overridden with "Read-Query-Timeout" in the broker.conf file.
 */
#define READ_QUERY_TIMEOUT 500000

/*
 *  TRUNCATE_DESCRIPTIONS - should the broker only save one line of description
 */
#ifndef TRUNCATE_DESCRIPTIONS
#define TRUNCATE_DESCRIPTIONS
#endif

/*
 *  FORK_ON_BULK - If defined, then the Broker writes the results of
 *  a bulk query to a temporary file, then forks a process to send
 *  the results over the network.
 */
#ifndef FORK_ON_BULK
#define FORK_ON_BULK
#endif

/*
 *  QM_RET_EMBED_ATT - define for the broker to return embedded
 *  attributes requested with the #attribute directive.
 *  See broker/query_man.c.
 */
#ifndef QM_RET_EMBED_ATT
#define QM_RET_EMBED_ATT
#endif

#define USE_GLIMPSE	1
#define USE_WAIS	1

/* ============ Return Values =========== */
/* we want to always redefine these values */
#undef ERROR
#undef FAIL
#undef SUCCESS
#undef TRUE
#undef FALSE

#define ERROR 	-1
#define FAIL 	0
#define SUCCESS	1
#define TRUE 	1
#define FALSE 	0


/*
 *  The Registry needs a 32-bit number for storage in on the file system.
 *  Define the num32 type as a 32-bit number as per the architecture.
 *  For example, on a DEC Alpha int's are 4 bytes and long's are 8 bytes.  
 */
#if SIZEOF_LONG == 4
typedef long num32;
#elif SIZEOF_INT == 4
typedef int num32;
#else
typedef long num32;		/* assume that long's are 32bit */
#endif
#define NUM32LEN sizeof(num32)

/* ============ Parser Modes =========== */
#define NO_MODE 	0
#define UPD_MODE 	1
#define DEL_MODE 	2
#define REF_MODE 	3
#define LIST_MODE 	4

/* ============ Collector Modes  =========== */
#define UPD 	0
#define DEL 	1
#define REF 	2

/* ============ Debug =========== */
#ifdef DEBUG
#undef DEBUG
#define DEBUG 99		/* Set to 0 to remove all messages */
#endif

/* Debugging/tracing levels */
#define DEBUG0		(DEBUG > 0)
#define DEBUG1 		(DEBUG > 1)
#define DEBUG2 		(DEBUG > 2)
#define DEBUG3 		(DEBUG > 3)
#define DEBUG9 		(DEBUG > 9)

/* ============ Indexer Definitions  =========== */
#define FULLI   	"Full"
#define INCRI   	"Incremental"
#define PEROBJI 	"Per-Object"

#define I_FULL    	123
#define I_INCR    	132
#define I_PER_OBJ 	231

/* ============ Default Definitions =========== */
#define COLLECT_RATE 	86400	/* 24 hrs in sec */
#define CLEAN_RATE 	43200	/* 12 hrs in sec */
#define REFRESH_RATE 	2419200	/* 1 month in sec */
#define Q_PORT 		8501	/* default port */

 /* ============ Configuration Tags =========== */
#define S_BRKDIR     	"Broker-Directory"
#define S_BRKHP     	"Broker-Home-Page"
#define S_BRKOFFLINE    "Collection-Only"
#define S_CLEANR     	"Clean-Rate"
#define S_COLLR      	"Collection-Rate"
#define S_DESC    	"Description-Tag"
#define S_RFR        	"Refresh-Rate"
#define S_PORT       	"Port"
#define S_GATHER       	"Gather"
#define S_LOGK       	"Log-Key"
#define S_TERSELOG      "Terse-Logging"
#define S_FASTSTART     "Fast-Start"
#define S_FCOLL      	"Collection-Time"
#define S_INDTP      	"Index-Type"
#define S_INDEXER      	"Indexer-Type"
#define S_DEDLIM     	"Dead-Entry-Limit"
#define S_MAXEV         "Event-Queue-Limit"
#define S_WEBS       	"Web-Server"
#define S_WEBPATH    	"Web-Path"
#define S_COLCONF    	"Collection-Configure-File"
#define S_PASSWD        "Admin-Password"
#define S_APROC         "Admin-Process"
#define S_RDQTO         "Read-Query-Timeout"

/* ============ Misc. Definitions =========== */
#define MAX_URL 	512
#define MAX_FN_SIZE 	4096

#define UFULL_U    	0
#define UPARTIAL_U 	1
#define CFULL_U    	2
#define CPARTIAL_U 	3
#define BAFULL_U    	4
#define BAPARTIAL_U 	5
#define BQFULL_U    	6
#define BQPARTIAL_U 	7

#define LIST_RG 	1

#define MAX_DEAD 	6000
#define MAX_QUERY 	2048

#define SWRITE(_S, _B, _L) \
	if (write(_S, _B, _L) == -1) { \
		log_errno("socket write"); \
		return ERROR; \
	}

/* ============ Module Names =========== */
#define STMGR    	"Storage Manager: "
#define REGIS    	"Registry: "
#define COLLECT  	"Collector: "
#define SCANNER  	"Scanner: "
#define PARSER   	"Parser: "
#define LOGG     	"Statistics Log:"

/* ============ Error definitions =========== */
#define FD_ERR   	"Unable to get FD"
#define OPEN_ERR 	"Unable to open file"
#define UNLINK_ERR 	"Unable to unlink file"
#define CLOSE_ERR 	"Unable to close file"
#define WRITE_ERR 	"Unable to write file"
#define READ_ERR 	"Unable to read file"
#define OBJECT_ERR 	"Object not found"
#define ENTRY_ERR  	"Insufficient data supplied"
#define TIME_ERR  	"Unable to get current time"
#define PARSE_ERR 	"Parse Error"

/* ============= Field Name definitions =========== */
#define UPDATE_A        "update-time"
#define MD5             "md5"
#define TTL       	"time-to-live"
#define LMT_A    	"last-modification-time"
#define REFRESH_A 	"refresh-rate"
#define GATH_HOST 	"gatherer-host"
#define GATH_NAME 	"gatherer-name"
#define GATH_VER  	"gatherer-version"
#define NESTED_FN 	"nested-filename"
#define OBJ_DESC  	"description"

#define TTL_S       	12	/* cache of attribute name lengths */
#define LMT_A_S    	22
#define REFRESH_A_S 	12
#define GATH_HOST_S 	13
#define GATH_NAME_S 	13
#define GATH_VER_S  	16
#define NESTED_FN_S 	15
#define OBJ_DESC_S  	11
#define UPDATE_A_S      11
#define MD5_S           3

#define MD5LEN          32

/* ============ Global Data Type ======== */
typedef num32 fd_t;

/* Gatherer Identifier */
#define MAX_GATHERER_ID         8192
typedef struct _broker_gatherer_id {
        int GID;        /* Identifying number for gid cache */
        char *gn;       /* Gatherer-Name */
        num32 gns;      /* Gatherer-Name size */
        char *gh;       /* Gatherer-Host */
        num32 ghs;      /* Gatherer-Host size */
        char *gv;       /* Gatherer-Version */
        num32 gvs;      /* Gatherer-Version size */
} GathererID;


/* An in-memory Registry entry */
typedef struct REG_T {
	/* Object Identifier information: MANDATORY */
	fd_t FD;		/* The OID Number */
	char *url;		/* The URL */
	num32 urls;		/* cache of URL size */
	time_t update_time;	/* The Update-Time */
	int GID;		/* GathererID Identifier */

	/* Extra information about the Object */
	char *md5;		/* The MD5: OPTIONAL */
	num32 md5s;		/* cache of md5 size */
	time_t ttl;		/* The Time-to-Live */
	time_t lmt;		/* The Last-Modification-Time */
	time_t refresh_rate;	/* The Refresh-Rate */
	num32 flag;		/* Flag for misc information */
	off_t rec_off;		/* Offset in the Registry file */
	char *desc;		/* One-line description of object */
	num32 descs;		/* cache of desc size */

	struct REG_T *next, *prev;	/* Doubly-linked list */
} reg_t;

#define LONGLEN 	sizeof(long)
#define INTLEN 		sizeof(int)
#ifndef TIMELEN
#define TIMELEN 	sizeof(time_t)
#endif
#define OFFLEN 		sizeof(off_t)
#define FDLEN 		sizeof(fd_t)


/* ------------ Registry Storage Manager Types and Macros ------- */
#define REGISTRY_MAGIC		0x1f3a2bec
#define REGISTRY_VERSION	1
#define SET_DELETED(x)		(x) |= 0x01
#define IS_DELETED(x)		((x) & 0x01)

#define ENTRY_DELETED		2
#define REGISTRY_EOF		3

typedef struct reghdr {
	num32 magic;
	num32 version;
	num32 nrecords;
	num32 nrecords_deleted;
	num32 nrecords_valid;
} REGISTRY_HEADER;

typedef struct regrechdr {
	num32 record_size;
	num32 magic;
	num32 flag;
} RECORD_HEADER;

/* ============ Query Manager Data Types ======== */
/* Types */
#define LOGICAL 	0
#define SELECT  	1
#define FUNCTION  	2
#define STRING		3

/* Ops */
#define AND     	10
#define OR      	11
#define NOT     	12
#define EXCEPT  	13

#define EXACT   	20
#define REGEX   	21
#define GTT     	22
#define LST     	23

/* query types */
#define UQUERY     	30
#define QBULK      	31
#define ABULK      	32
#define ADMIN      	33
#define QDELETE    	34

/*action types */
#define COLLECTION 	40
#define CLEANING   	41
#define RCOMP      	42

/* Admin ops */
#define AD_COLLECT  	50
#define AD_CLEAN    	51
#define AD_REFRESH  	52
#define AD_UNZIP    	53

typedef struct QLIST {
	int type;
	int op;
	char *rlist;
	char *llist;
} qlist_t;

typedef struct ARG {
	int		type;
	void		*ptr;
	struct ARG	*next;
} arg_t;

/* ============ Event Manager Data Types ======== */
#define NUM_PRTY 4

typedef struct EVENT {
	int type;
	char queri[MAX_QUERY];
	int sock;
	time_t qtime;
	struct EVENT *next;
} event_t;

typedef struct PRTY {
	event_t *top;
	event_t *tail;
	int qnum;
} prty_t;

typedef struct EVNTQ {
	prty_t pri[NUM_PRTY];
} eventq_t;

#define USEREV    	"#USER"
#define BULKEV    	"#BULK"
#define ADMINEV   	"#ADMIN"
#define USEREV_S  	5
#define BULKEV_S  	5
#define ADMINEV_S 	6

/*  ========== BROKER RESULT CODES ========== */

#define BR_USER_MSG		101
#define BR_END_RESULTS		103
#define BR_URL			120
#define BR_OPAQUE_DATA		122
#define BR_DESCRIPTION		124
#define BR_SUMMARY_URL		125
#define BR_BROKER_URL		126
#define BR_ATTRIBUTE		127
#define BR_END_OBJECT		130

/* ========== ERROR Messages ========== */
#define ERR_MSG "111 - Indexer ERROR: The Indexer does not support all queries.\n"
#define ERR_SIZE 63

#define FILEONLY_WARN "101 - Warning: The Indexer does not file only listings.\n"
#define FILEONLY_WARN_S strlen(FILEONLY_WARN)

#define IND_FAIL "111 - Indexer ERROR: Could not complete query.\n"
#define IND_FAIL_S 47

#define LOADERR "111 - Broker is too heavily loaded.  Please try again later.\n"
#define LOADERR_S 60

#define PARSERR "111 - PARSE ERROR: Invalid Query.\n"
#define PARSERR_S 34

#define ADMINERR "Last Command was UNsuccessful.\n"
#define ADMINERR_S 31

#define ADMINSUC "Last Command was Successful.\n"
#define ADMINSUC_S 29

#define BULK_SUC  "000 - Bulk Object Transfer Successfully Initiated.\n"
#define BULK_END  "099 - Bulk Object Transfer Completed.\n"
#define BULK_ERR  "400 - Bulk Object Transfer Error.\n"

#define QMVERSION "200 - Harvest Broker Interface Version 0.3.\n"
#define PIPECHK   "200 - Testing the socket to the client.\n"

/* ============ INDEXER ============== */
struct indexing_routines {
	char *indexer_type;
	int (*IND_Index_Start) ();
	int (*IND_Index_Flush) ();
	int (*IND_New_Object) ();
	int (*IND_Destroy_Obj) ();
	int (*IND_Index_Full) ();
	int (*IND_Index_Incremental) ();
	int (*IND_initialize) ();
	int (*IND_config) ();
	int (*IND_do_query) ();
	void (*IND_Init_Flags) ();
	void (*IND_Set_Flags) ();
};


/* ============ Prototyping ============= */

#ifndef _PARAMS
#if defined(__STDC__) || defined(__cplusplus) || defined(__STRICT_ANSI__)
#define _PARAMS(ARGS) ARGS
#else				/* Traditional C */
#define _PARAMS(ARGS) ()
#endif				/* __STDC__ */
#endif				/* _PARAMS */

/* stor_man.c - Storage Manager */
int 	SM_Init _PARAMS((void));		/* Initalize */
fd_t 	SM_Create_Obj _PARAMS((void));		/* Create Object */
int 	SM_Destroy_Obj _PARAMS((fd_t));		/* Delete Object */
int 	SM_Exist_Obj _PARAMS((fd_t));		/* Exist Object? */
FILE *	SM_Read_Obj _PARAMS((fd_t));		/* Read Object Data */
FILE *	SM_Write_Obj _PARAMS((fd_t));		/* Write Object Data */
char *  SM_Get_Obj_URL _PARAMS((fd_t));		/* URL to Read Object Data */

/* main.c */
FILE *StMgr_Get_Object _PARAMS((fd_t, char *));
void sigdie _PARAMS((int));
void sigreap _PARAMS((int));
int Config_Broker _PARAMS((char *));
int Initialize_Com _PARAMS((void));
int set_str _PARAMS((char *, char *));
int set_rate _PARAMS((char *, int));
int Broker_Shutdown _PARAMS((void));
int Do_Compression _PARAMS((void));
int Do_Collection _PARAMS((void));
int Do_Cleaning _PARAMS((void));

/* registry.c */
int RG_Compress _PARAMS((void));
int RG_FD_Exists _PARAMS((fd_t));
int RG_Init _PARAMS((void));
int RG_Register _PARAMS((reg_t *));
int RG_Unregister _PARAMS((reg_t *));
int RG_Free_Entry _PARAMS((reg_t *));
int RG_List_Reg _PARAMS((void));
int RG_Build_Registry _PARAMS((void));
int RG_Cleaner _PARAMS((void));
int RG_Clean_Entry _PARAMS((reg_t *));
int RG_Count_Reg _PARAMS((void));
int RG_bulk_query _PARAMS((int, time_t));
int RG_Comp_Reg _PARAMS((void));
int RG_Free_Registry _PARAMS((void));
char *RG_Get_URL _PARAMS((fd_t));
reg_t *RG_Object_Search_Entry _PARAMS((reg_t *));
reg_t *RG_Object_Search_ByURL _PARAMS((char *));
reg_t *RG_Get_Entry _PARAMS((fd_t));
void RG_Registry_Shutdown _PARAMS((void));
int RG_Sync_Registry _PARAMS((void));

/* gid.c */
int RG_gid_register _PARAMS((GathererID *));
void RG_gid_init _PARAMS((void));
GathererID *RG_gid _PARAMS((int));
GathererID *RG_gid_new _PARAMS((char *, char *, char*));
void RG_gid_free _PARAMS((GathererID *));
void RG_gid_destroy _PARAMS((void));

/* collector.c */
FILE *COL_UPD_Obj_begin _PARAMS((reg_t *));
int COL_Fill_Entry _PARAMS((reg_t *));
int COL_UPD_Obj_end _PARAMS((reg_t *));
int COL_DEL_Obj _PARAMS((reg_t *));
int COL_REF_Obj _PARAMS((reg_t *));
int COL_Save_Att _PARAMS((AVPair *, reg_t *));
int COL_RM_Dup_Obj _PARAMS((reg_t *));
int COL_Do_Collection _PARAMS((void));
int COL_reset_update _PARAMS((char *, int, time_t));
time_t COL_get_last_update _PARAMS((char *, int));
char *COL_Normalize_Name _PARAMS((char *));
char *COL_Init_Connect _PARAMS((char *, int, int, time_t, char *));
char *COL_Get_UInput _PARAMS((int, time_t));
char *COL_Get_CInput _PARAMS((int, time_t));
char *COL_Get_BInput _PARAMS((int, char *, int, time_t));
FILE *COL_Create_Read_Pipe _PARAMS((char *));
void COL_Close_Read_Pipe _PARAMS((FILE *));

/* parser.y */
int P_parse_input _PARAMS((char *, int));
int P_parse_command _PARAMS((FILE *, int));
int P_parse_object _PARAMS((int));
int P_get_next_char _PARAMS((FILE *));

/* log.c */
int LOG_Init _PARAMS((void));
int LOG_rotate _PARAMS((void));
int LOG_turn_on _PARAMS((unsigned int));
int LOG_turn_off _PARAMS((unsigned int));
int LOG_reset_master _PARAMS((void));
int LOG_flush_log _PARAMS((void));
int LOG_close_log _PARAMS((void));
FILE *LOG_fptr _PARAMS((void));
unsigned int LOG_match_name _PARAMS((char *));

/* query_man.c */
int QM_query_manager _PARAMS((event_t *));
int QM_free_qlist _PARAMS((qlist_t *));
int QM_send_bulk_begin _PARAMS((int));
int QM_send_bulk_err _PARAMS((int));
int QM_send_bulk_end _PARAMS((int));
int QM_send_bulk_fd _PARAMS((fd_t, FILE *, reg_t *));
int QM_user_object _PARAMS((int, fd_t, int, char **));
int QM_user_done _PARAMS((int, int));
int QM_user_warning _PARAMS((int, char *));
int QM_add_attribute _PARAMS((char *));
int QM_free_attributes _PARAMS((void));


/* brkutil.c */
int UTIL_Make_Lower _PARAMS((char *));
char *UTIL_make_admin_filename _PARAMS((char *));
time_t UTIL_Get_Time _PARAMS((void));
int do_IND_Index_Start _PARAMS((void));		/* indexer */
int do_IND_Index_Flush _PARAMS((void));
int do_IND_New_Object _PARAMS((reg_t *));
int do_IND_Destroy_Obj _PARAMS((reg_t *));
int do_IND_Index_Full _PARAMS((void));
int do_IND_Index_Incremental _PARAMS((void));
int do_IND_initialize _PARAMS((void));
int do_IND_config _PARAMS((char *, char *));
int do_IND_do_query _PARAMS((qlist_t *, int, int, time_t));
void do_IND_Init_Flags _PARAMS((void));
void do_IND_Set_Flags _PARAMS((char *, char *));

/* event.c */
int EV_Init _PARAMS((void));
int EV_add_sevent _PARAMS((int));
int EV_add_aevent _PARAMS((int));
int EV_enq _PARAMS((event_t *, int));
int EV_Do_Event _PARAMS((void));
event_t *EV_deq _PARAMS((void));

/* admin.c */
int AD_add_object _PARAMS((char *));
int AD_remove_name _PARAMS((char *));
int AD_log_on _PARAMS((char *));
int AD_log_off _PARAMS((char *));
int AD_restart _PARAMS((void));
int AD_do_stats _PARAMS((void));
void AD_run_admin_process _PARAMS((void));

/* stor_reg.c */
int init_registry_file _PARAMS((void));
int set_registry_file _PARAMS((char *));
void finish_registry_file _PARAMS((void));
int read_record _PARAMS((reg_t *));
int write_record _PARAMS((reg_t *));
int delete_record _PARAMS((num32));
REGISTRY_HEADER *read_header _PARAMS((void));
int write_header _PARAMS((REGISTRY_HEADER *rhdr));
int get_record _PARAMS((reg_t *));
int append_new_record _PARAMS((reg_t *));
int remove_record _PARAMS((reg_t *));
int replace_record _PARAMS((reg_t *));
int read_record_hdr _PARAMS((RECORD_HEADER *));
int write_record_hdr _PARAMS((RECORD_HEADER *));
off_t set_registry_mark _PARAMS((void));
off_t restore_registry_mark _PARAMS((void));

/* select_loop.c */
int select_loop _PARAMS((int, int, int));

/* main.c */
int verify_exe _PARAMS((char *));
int verify_path _PARAMS((char *));

#endif
