
/*
 *           PVM 3.2:  Parallel Virtual Machine System 3.2
 *               University of Tennessee, Knoxville TN.
 *           Oak Ridge National Laboratory, Oak Ridge TN.
 *                   Emory University, Atlanta GA.
 *      Authors:  A. L. Beguelin, J. J. Dongarra, G. A. Geist,
 *    W. C. Jiang, R. J. Manchek, B. K. Moore, and V. S. Sunderam
 *                   (C) 1992 All Rights Reserved
 *
 *                              NOTICE
 *
 * Permission to use, copy, modify, and distribute this software and
 * its documentation for any purpose and without fee is hereby granted
 * provided that the above copyright notice appear in all copies and
 * that both the copyright notice and this permission notice appear in
 * supporting documentation.
 *
 * Neither the Institutions (Emory University, Oak Ridge National
 * Laboratory, and University of Tennessee) nor the Authors make any
 * representations about the suitability of this software for any
 * purpose.  This software is provided ``as is'' without express or
 * implied warranty.
 *
 * PVM 3.2 was funded in part by the U.S. Department of Energy, the
 * National Science Foundation and the State of Tennessee.
 */

/*
 *	lpvmmimd.c
 *
 *	Libpvm core for MPP environment.
 *
$Log: lpvmmimd.c,v $
 * Revision 1.1  1993/08/30  23:26:48  manchek
 * Initial revision
 *
 */

#include <stdio.h>
#include <rpc/types.h>
#include <rpc/xdr.h>
#include <sys/stat.h>
#ifdef IMA_CM5
#include <unistd.h>
#endif
#include <fcntl.h>
#ifdef	SYSVSTR
#include <string.h>
#else
#include <strings.h>
#endif
#include <errno.h>
#ifdef IMA_PGON
#include <nx.h>
#endif
#ifdef IMA_I860
#include <cube.h>
#endif
#include <pvm3.h>
#include "global.h"
#include "tdpro.h"
#include "ddpro.h"
#include "pvmalloc.h"
#include "pvmfrag.h"
#include "pvmumbuf.h"
#include "listmac.h"
#include "pvmmimd.h"
#include "bfunc.h"

/* task debug mask */

#define	TDMPACKET	1		/* packet tracing */
#define	TDMMESSAGE	2		/* message tracing */


char *getenv();

extern struct encvec *enctovec();



/***************
 **  Globals  **
 **           **
 ***************/

extern int errno;					/* from libc */
extern char *sys_errlist[];
extern int sys_nerr;

extern int pvmrbufmid;				/* from pack.c */
extern int pvmsbufmid;				/* from pack.c */

char *pvm_errlist[] = {					/* error messages for -pvm_errno */
		"Error 0",
		"Error 1",
	"Bad parameter",
	"Count mismatch",
		"Error 4",				/* not used */
	"End of buffer",
	"No such host",
	"No such file",
		"Error 8",				/* not used */
		"Error 9",				/* not used */
	"Malloc failed",
		"Error 11",				/* not used */
	"Can't decode message",
		"Error 13",				/* not used */
	"Can't contact local daemon",
	"No current buffer",
	"No such buffer",
	"Null group name",
	"Already in group",
	"No such group",
	"Not in group",
	"No such instance",
	"Host failed",
	"No parent task",
	"Not implemented",
	"Pvmd system error",
	"Version mismatch",
	"Out of resources",
	"Duplicate host",
	"Can't start pvmd",
	"Already in progress",
	"No such task",
	"No such entry",
	"Duplicate entry",
};

int pvmautoerr = 1;						/* whether to auto print err msgs */
int pvmcouttid = 0;						/* child stdout dst and code */
int pvmcoutcod = 0;
int pvmctrctid = 0;						/* child trace dst and code */
int pvmctrccod = 0;
int pvmfrgsiz = MAXFRAGSIZE;			/* message frag length (to pack) */
int pvmmyndf = 0;						/* host native data enc, init XDR */
int pvmmyptid = -1;						/* parent task id */
int pvmmytid = -1;						/* this task id */
int pvmmyupid = -1;						/* process unix pid */
int pvmudpmtu = MAXFRAGSIZE;			/* max fragment size */
int pvmouttid = 0;						/* stdout dst and code */
int pvmoutcod = 0;
int pvmtrctid = 0;						/* trace dst and code */
int pvmtrccod = 0;


#if defined(IMA_PGON) || defined(IMA_I860)

/* fragment on async send queue */
struct tmpfrag {
	struct frag *tf_fp;					/* data location */
	long tf_mid;						/* msg handle */
};

#endif

#if defined(IMA_CM5)

struct tmpfrag {
	struct frag *tf_fp;
	CMMD_mcb tf_mid;
};

#endif

struct tmpfrag outfrags[NUMSMIDS];		/* fragments queued by async send */
int nextsmid = 0;						/* index of current isend mid */

long my_ptype = 0;						/* my process type */
long my_host = -1;						/* my host node */
int my_node = -1;						/* my node number */
int mysetpart = 0;						/* host field & set field */
int pvm_errno = 0;						/* last libpvm error code */
int pvm_nerr = sizeof(pvm_errlist)
			/sizeof(pvm_errlist[0]);	/* exported num of errors */
struct umbuf *pvmrxlist = 0;				/* not-recvd msg list */
int pvmtidhmask = TIDHOST;					/* mask for host field of tids */
int tidnmask = TIDNODE;					/* mask for node field of tids */
int tidtmask = TIDPTYPE;				/* mask for proc type field of tids */


/***************
 **  Private  **
 **           **
 ***************/

static char rcsid[] = "$Id: lpvmmimd.c,v 1.1 1993/08/30 23:26:48 manchek Exp $";
static int debugmask = 0;				/* which debugging info */
static char etext[512];					/* scratch for error log */
static struct umbuf *rxfrag = 0;		/* not-assembled incm msgs */
static struct frag *rxbuf;				/* buffer for incoming packets */

#if defined(IMA_PGON) || defined(IMA_I860)
static long irecvmid = -1;				/* msg ID returned by irecv() */
#endif

#ifdef IMA_CM5
static CMMD_mcb irecvmid;				/* msg handle */
#endif

static int recvmask = 0x80000000 + (1 << PMTDN) + (1 << PMTNN)
						+ (1 << PMTMCAST);
								/* accept messages from pvmd or other nodes */
static int pvmrouteopt = PvmDontRoute;	/* task-task routing style */


/**************************
 **  Internal Functions  **
 **                      **
 **************************/

/*	bailout()
*
*	Called by low-level stuff in f.e. frag.c.  Don't really want to
*	bail in libpvm.
*/

void
pvmbailout(n)
	int n;
{
	n = n;	/* ayn rand was here */
}


/*	pvmlogerror()
*
*	Log a libpvm error message.  Prepends a string identifying the task.
*/

pvmlogerror(s)
	char *s;
{
	if (pvmmytid == -1)
		fprintf(stderr, "libpvm [pid%d]: %s", pvmmyupid, s);
	else
		fprintf(stderr, "libpvm [t%x]: %s", pvmmytid, s);
}


/*	pvmlogperror()
*
*	Log a libpvm error message.  Prepends a string identifying the
*	task and appends the system error string for _errno.
*/

pvmlogperror(s)
	char *s;
{
	char *em;

	em = ((errno >= 0 && errno < sys_nerr)
		? sys_errlist[errno] : "Unknown Error");
	if (pvmmytid == -1)
		fprintf(stderr, "libpvm [pid%d]: %s: %s\n", pvmmyupid, s, em);
	else
		fprintf(stderr, "libpvm [t%x]: %s: %s\n", pvmmytid, s, em);
}


hdump(p, n, pad)
	char *p;	/* bytes */
	int n;		/* length */
	char *pad;
{
	int i;
	pad = pad ? pad : "";
	for (i = 0; n-- > 0; i = ++i & 15) {
		fprintf(stderr, "%s%02x%s",
			i ? "" : pad,
			0xff & *p++,
			n && i != 15 ? " " : "\n");
	}
}


/*	mroute()
*
*	Move message frags between task and pvmd.
*	Returns when
*		outgoing message (if any) fully sent
*		(if block is true) at least one message fully received
*	Returns >=0 the number of complete messages downloaded, or
*	negative on error.
*/

int
mroute(mid, dtid, code, block)
	int mid;		/* message */
	int dtid;		/* dest */
	int code;		/* type code */
	int block;		/* get at least one message */
{
	struct umbuf *txup;			/* tx message or null */
	struct frag *txfp = 0;		/* cur tx frag or null */
	int gotem = 0;				/* count complete msgs downloaded */
	struct umbuf *rxup;			/* rx message */
	struct frag *fp;
	char *cp;
	int src;
	int dst;
	int ff;
	char *txcp = 0;				/* point to remainder of txfp */
	int len = 0;				/* len of txfp */
	long node;					/* destination node */
	long ptype;					/* destination process type */
	long mtype;					/* message type */
	long info[8];				/* info about pending message */
	int err = 0;
	int srcnode = -1;
	int i;

	if (txup = midtobuf(mid)) {
		txfp = txup->ub_frag->fr_link;
		if (txfp->fr_buf) {
			int mask = pvmtidhmask;		/* host */

#if defined(IMA_PGON) || defined(IMA_CM5)
			mask |= tidtmask;				/* process type */
#endif

			if (TIDISNODE(dtid) && (dtid & mask) == (pvmmytid & mask)) {
				node = dtid & tidnmask;
				ptype = my_ptype;			/* send to node directly */
				mtype = PMTNN;
			} else {
				node = my_host;
				ptype = PVMDPTYPE;			/* send to pvmd first */
				mtype = PMTND;
			}
		} else
			txfp = 0;
	}

	do {

		if ((!txfp && block) || (err = ASYNCDONE(irecvmid))) {

			if (!txfp && block)
				ASYNCWAIT(irecvmid);
			if (err < 0) {
				pvmlogperror("mroute() ASYNCDONE");
				return PvmSysErr;
			}

			rxbuf->fr_len = MSGSIZE(irecvmid);
			srcnode = MSGSENDER(irecvmid);

#ifdef IMA_CM5
			CMMD_free_mcb(irecvmid);
#endif

			fp = rxbuf;
			rxbuf = fr_new(pvmfrgsiz);
			irecvmid = ASYNCRECV(rxbuf->fr_dat, rxbuf->fr_max);

			/* fp->fr_len = fp->fr_max; */
			cp = fp->fr_dat;
			dst = pvmget32(cp);
			src = pvmget32(cp + 4);
			if (dst != pvmmytid && !(dst & TIDGID)) {
				sprintf(etext,
					"mroute() pkt src t%x node %d dst t%x scrapped (not us)\n",
					src, srcnode, dst);
				pvmlogerror(etext);
				continue;
			}
			ff = pvmget8(cp + 12);
			fp->fr_len -= TDFRAGHDR;
			fp->fr_dat += TDFRAGHDR;
/*
			fprintf(stderr, "mroute() frag src t%x len %d ff %d\n",
					src, fp->fr_len, ff);
*/
			if (debugmask & TDMPACKET) {
				sprintf(etext, "mroute() src t%x node %d len %d dst t%x\n",
					src, srcnode, fp->fr_len, dst);
				pvmlogerror(etext);
			}
	/*
	* if start of message, make new umbuf, add to frag pile
	*/
			if (ff & FFSOM) {
				cp += TDFRAGHDR;
				fp->fr_len -= TTMSGHDR;
				fp->fr_dat += TTMSGHDR;
				rxup = midtobuf(umbuf_new());
				rxup->ub_cod = pvmget32(cp);
				rxup->ub_enc = pvmget32(cp + 4);
				rxup->ub_src = src;
				LISTPUTBEFORE(rxfrag, rxup, ub_link, ub_rlink);
			}

	/* locate frag's message */

			for (rxup = rxfrag->ub_link; rxup != rxfrag;
					rxup = rxup->ub_link)
				if (rxup->ub_src == src)
					break;

			if (rxup == rxfrag) {	/* uh oh, no message for it */
				pvmlogerror("mroute() frag with no message\n");
				fr_unref(fp);

			} else {
				LISTPUTBEFORE(rxup->ub_frag, fp, fr_link, fr_rlink);
				rxup->ub_len += fp->fr_len;
	/*
	* if end of message, move to rxlist and count it
	*/
				if (ff & FFEOM) {
					LISTDELETE(rxup, ub_link, ub_rlink);
					rxup->ub_codef = enctovec(rxup->ub_enc);
					LISTPUTBEFORE(pvmrxlist, rxup, ub_link, ub_rlink);
					gotem++;
				}
			}
		}


		if (txfp) {
			/* has msg to send */

			txcp = txfp->fr_dat;
			len = txfp->fr_len;
	/*
	* if this is first frag, prepend t-t header
	*/
			ff = 0;
			if (txfp->fr_rlink == txup->ub_frag) {
				txcp -= TTMSGHDR;
				len += TTMSGHDR;
				pvmput32(txcp, code);
				pvmput32(txcp + 4, txup->ub_enc);
				ff = FFSOM;
			}
			if (txfp->fr_link == txup->ub_frag)
				ff |= FFEOM;
	/*
	* prepend t-d header
	*/
			txcp -= TDFRAGHDR;
			pvmput32(txcp, dtid);
			pvmput32(txcp + 4, pvmmytid);
			pvmput32(txcp + 8, len);
			pvmput8(txcp + 12, ff);
			len += TDFRAGHDR;
			if (debugmask & TDMPACKET) {
				sprintf(etext, "mroute() dst t%x len %d ptype=%ld node=%ld\n",
					dtid, txfp->fr_len, ptype, node);
				pvmlogerror(etext);
			}

			/* check any pending sends; free data if send is completed */

			if (nextsmid == NUMSMIDS)
				nextsmid = 0;
			i = nextsmid;
			while (outfrags[i].tf_mid > 0 &&
			ASYNCDONE(outfrags[i].tf_mid) > 0) {
				fr_unref(outfrags[i].tf_fp);
#ifdef IMA_CM5
				CMMD_free_mcb(outfrags[i].tf_mid);
#endif
				outfrags[i++].tf_mid = -1;
			}

			if (outfrags[nextsmid].tf_mid < 0) {
				if ((outfrags[nextsmid].tf_mid =
				ASYNCSEND(mtype, txcp, len, node, ptype)) < 0) {
					pvmlogperror("mroute() asyncsend");
					return PvmSysErr;
				}
				txfp->fr_ref++;		/* don't free data 'til frag's sent */
				outfrags[nextsmid++].tf_fp = txfp;
			} else {
				/* too many isend's outstanding; use csend */
				if (PVMCSEND(mtype, txcp, len, node, ptype) < 0) {
					pvmlogperror("mroute() csend");
					return PvmSysErr;
				}
			}
			txfp = txfp->fr_link;
			if (!txfp->fr_buf)
				txfp = 0;
		}

	} while (txfp || (block && !gotem));

	return gotem;
}


/*	msendrecv()
*
*	Single op to send a system message (usually to our pvmd) and get
*	the reply.
*	Returns message handle or negative if error.
*/

int
msendrecv(other, code)
	int other;				/* dst, src tid */
	int code;				/* message code */
{
	int cc;
	struct umbuf *up;

	if (pvmsbufmid <= 0)
		return PvmNoBuf;

	/* send code to other */
	if (debugmask & TDMMESSAGE) {
		sprintf(etext, "msendrecv() to t%x code %d\n", other, code);
		pvmlogerror(etext);
	}
	if ((cc = mroute(pvmsbufmid, other, code, 1)) < 0)
		return cc;

	/* recv code from other */
	for (up = pvmrxlist->ub_link; 1; up = up->ub_link) {
		if (up == pvmrxlist) {
			up = up->ub_rlink;
			if ((cc = mroute(0, 0, 0, 1)) < 0)
				return cc;
			up = up->ub_link;
		}

		if (debugmask & TDMMESSAGE) {
			sprintf(etext, "msendrecv() cmp from t%x code %d\n",
					up->ub_src, up->ub_cod);
			pvmlogerror(etext);
		}
		if (up->ub_src == other && up->ub_cod == code)
			break;
	}
	LISTDELETE(up, ub_link, ub_rlink);
	if (pvmrbufmid > 0)
		umbuf_free(pvmrbufmid);
	pvmrbufmid = 0;
	if (cc = pvm_setrbuf(up->ub_mid))
		return cc;
	return up->ub_mid;
}


/*	pvmbeatask()
*
*	Initialize libpvm, config process as a task.
*	This is called as the first step of each libpvm function so no
*	explicit initialization is required.
*
*	Returns 0 if okay, else error code.
*/

int
pvmbeatask()
{
	int pvminfo[SIZEHINFO];		/* proto, hostpart, ptid, MTU, NDF */
	long rmid = -1;				/* msg ID returned by irecv() */
	int i;

	if (pvmmytid != -1)
		return 0;

#if defined(IMA_PGON) || defined(IMA_I860)
	if ((rmid = _irecv(PMTHOST, pvminfo, sizeof(pvminfo))) < 0) {
		pvmlogperror("beatask() recv pvminfo");
		return PvmSysErr;
	}
	if ((my_host = _myhost()) < 0)
		pvmlogperror("beatask() no controlling process");
#endif

#ifdef IMA_CM5
	CMMD_enable_host();
	CMMD_receive_bc_from_host(pvminfo, sizeof(pvminfo));
	CMMD_reset_partition_size(pvminfo[0]);
	if ((my_node = CMMD_self_address()) >= pvminfo[0])
		exit(0);		/* no task to spawn on this node  */

	CMMD_fset_io_mode(stdout, CMMD_independent);
	CMMD_fset_io_mode(stderr, CMMD_independent);
	my_host = CMMD_host_node();
#endif

	pvmmyupid = getpid();

/*
sprintf(etext, "pvminfo: %d  %d  %d  %d  %d  node=%d\n", pvminfo[0], pvminfo[1],  pvminfo[2], pvminfo[3], pvminfo[4], my_node);
pvmlogerror(etext);
fflush(stdout);
*/

#ifdef IMA_PGON
	if ((my_ptype = myptype()) == INVALID_PTYPE)
		pvmlogerror("beatask() no process type\n");
	my_node = _mynode();
#endif
#ifdef IMA_I860
	my_node = _mydirect();
	if (_csend(PMTPHYS, &my_node, sizeof(int), my_host, PVMDPTYPE) < 0)
		pvmlogperror("beatask() can't send to host");
#endif

	/*
	*	initialize received-message list and fragment reassembly list
	*/

	rxfrag = TALLOC(1, struct umbuf, "umb");
	BZERO((char*)rxfrag, sizeof(struct umbuf));
	rxfrag->ub_link = rxfrag->ub_rlink = rxfrag;

	pvmrxlist = TALLOC(1, struct umbuf, "umb");
	BZERO((char*)pvmrxlist, sizeof(struct umbuf));
	pvmrxlist->ub_link = pvmrxlist->ub_rlink = pvmrxlist;

	for (i = 0; i < NUMSMIDS; i++)
		outfrags[i].tf_mid = -1;

#if defined(IMA_PGON) || defined(IMA_I860)
	_msgwait(rmid);
#endif

#if defined(IMA_PGON) || defined(IMA_I860)
	if (pvminfo[0] != TDPROTOCOL) {
		sprintf(etext, "beatask() t-d protocol mismatch (%d/%d)\n",
			TDPROTOCOL, pvminfo[0]);
		pvmlogerror(etext);
		return PvmSysErr;
	}
#endif

	mysetpart = pvminfo[1];
	pvmmyptid = pvminfo[2];
	pvmudpmtu = pvminfo[3];
	pvmmyndf = pvminfo[4];
	pvmmytid = mysetpart + my_node;
	pvmfrgsiz = pvmudpmtu;
	/* post a receive for mroute() */
	rxbuf = fr_new(pvmfrgsiz);
	irecvmid = ASYNCRECV(rxbuf->fr_dat,rxbuf->fr_max);

	return 0;
}


int
pvmendtask()
{
	if (pvmmytid != -1) {
		pvmmytid = -1;
	}

#ifdef IMA_CM5
	CMMD_all_msgs_wait();
#endif

	/* XXX free rxfrag and rxlist */

	return 0;
}


/************************
 **  Libpvm Functions  **
 **                    **
 ************************/


int
pvm_getopt(what)
	int what;
{
	int rc = 0;
	int err = 0;

	switch (what) {
	case PvmRoute:
		rc = pvmrouteopt;
		break;

	case PvmDebugMask:
		rc = debugmask;
		break;

	case PvmAutoErr:
		rc = pvmautoerr;
		break;

	case PvmOutputTid:
		rc = pvmcouttid;
		break;

	case PvmOutputCode:
		rc = pvmcoutcod;
		break;

	case PvmTraceTid:
		rc = pvmctrctid;
		break;

	case PvmTraceCode:
		rc = pvmctrccod;
		break;

	case PvmFragSize:
		rc = pvmfrgsiz;
		break;

	default:
		err = 1;
		break;
	}
	if (err)
		return lpvmerr("pvm_getopt", PvmBadParam);
	return rc;
}


int
pvm_setopt(what, val)
	int what;
	int val;
{
	int rc = 0;
	int err = 0;

	switch (what) {
	case PvmRoute:
		switch (val) {
		case PvmDontRoute:
		case PvmAllowDirect:
		case PvmRouteDirect:
			rc = pvmrouteopt;
			pvmrouteopt = val;
			break;

		default:
			err = 1;
			break;
		}
		break;

	case PvmDebugMask:
		rc = debugmask;
		debugmask = val;
		break;

	case PvmAutoErr:
		rc = pvmautoerr;
		pvmautoerr = val;
		break;

	case PvmOutputTid:
		if (val && val != pvmmytid
		&& (val != pvmouttid || pvmcoutcod != pvmoutcod))
			err = 1;
		else {
			rc = pvmcouttid;
			pvmcouttid = val;
		}
		break;

	case PvmOutputCode:
		if (pvmcouttid && pvmcouttid != pvmmytid && val != pvmoutcod)
			err = 1;
		else {
			rc = pvmcoutcod;
			pvmcoutcod = val;
		}
		break;

	case PvmTraceTid:
		if (val && val != pvmmytid
		&& (val != pvmtrctid || pvmctrccod != pvmtrccod))
			err = 1;
		else {
			rc = pvmctrctid;
			pvmctrctid = val;
		}
		break;

	case PvmTraceCode:
		if (pvmctrctid && pvmctrctid != pvmmytid && val != pvmtrccod)
			err = 1;
		else {
			rc = pvmctrccod;
			pvmctrccod = val;
		}
		break;

	case PvmFragSize:
/*
		if (val < TDFRAGHDR + TTMSGHDR + 4 || val > 1048576)
*/
		if (val < TDFRAGHDR + TTMSGHDR + 4 || val > MAXFRAGSIZE)

			err = 1;
		else {
			rc = pvmfrgsiz;
			pvmfrgsiz = val;
		}
		break;

	default:
		err = 1;
		break;
	}
	if (err)
		return lpvmerr("pvm_setopt", PvmBadParam);
	return rc;
}


int
pvm_perror(s)
	char *s;
{
	if (pvmmytid == -1)
		fprintf(stderr, "libpvm [pid%d]: ", pvmmyupid);
	else
		fprintf(stderr, "libpvm [t%x]: ", pvmmytid);
	fprintf(stderr, "%s: %s\n",
		(s ? s : "(null)"),
		(pvm_errno <= 0 && pvm_errno > -pvm_nerr
				? pvm_errlist[-pvm_errno] : "Unknown Error"));
	return 0;
}


int
pvm_getfds(fds)
	int **fds;
{
	return PvmNotImpl;
}


/*
* Obsolete Functions
*/

int
pvm_advise(what)
	int what;
{
	return pvm_setopt(PvmRoute, what);
}


int
pvm_setdebug(mask)
	int mask;
{
	return pvm_setopt(PvmDebugMask, mask);
}


