
/*
 *           PVM 3.2:  Parallel Virtual Machine System 3.2
 *               University of Tennessee, Knoxville TN.
 *           Oak Ridge National Laboratory, Oak Ridge TN.
 *                   Emory University, Atlanta GA.
 *      Authors:  A. L. Beguelin, J. J. Dongarra, G. A. Geist,
 *    W. C. Jiang, R. J. Manchek, B. K. Moore, and V. S. Sunderam
 *                   (C) 1992 All Rights Reserved
 *
 *                              NOTICE
 *
 * Permission to use, copy, modify, and distribute this software and
 * its documentation for any purpose and without fee is hereby granted
 * provided that the above copyright notice appear in all copies and
 * that both the copyright notice and this permission notice appear in
 * supporting documentation.
 *
 * Neither the Institutions (Emory University, Oak Ridge National
 * Laboratory, and University of Tennessee) nor the Authors make any
 * representations about the suitability of this software for any
 * purpose.  This software is provided ``as is'' without express or
 * implied warranty.
 *
 * PVM 3.2 was funded in part by the U.S. Department of Energy, the
 * National Science Foundation and the State of Tennessee.
 */

/*
 * pvmhost.c
 *
 * Special task that relays messages between pvmd and node tasks.
 * It runs on the frontend and connects to pvmd via sockets, but
 * communicates with the nodes using CMMD send/receive.
 *
 *
 * Jul 10 16:50:52 1993
 * 		changed mc_dsts[] to short from int.
 *
$Log: pvmhost.c,v $
 * Revision 1.1  1993/08/30  23:35:09  manchek
 * Initial revision
 *
 */
#include <sys/types.h>
#include <sys/time.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <errno.h>
#include <cm/cmmd.h>

#include "global.h"
#include "tdpro.h"
#include "protoglarp.h"
#include "pvmalloc.h"
#include "pvmfrag.h"
#include "pvmmimd.h"
#include "bfunc.h"

#define TVCLEAR(tvp)    ((tvp)->tv_sec = (tvp)->tv_usec = 0)
#ifndef min
#define min(a,b)    ((a)<(b)?(a):(b))
#endif

char *getenv();

/* from lpvm.c */
extern int errno;
extern int pvmmyptid;
extern int pvmmytid;
extern int pvmudpmtu;
extern int pvmmyndf;
extern int pvmtidhmask;

static char rcsid[] = "$Id: pvmhost.c,v 1.1 1993/08/30 23:35:09 manchek Exp $";
static char etext[512];         /* scratch for error log */
static int tidnmask = TIDNODE;
static int tidtmask = TIDPTYPE;


main(argc, argv)
	int argc;
	char **argv;
{
	int pvminfo[SIZEHINFO];    /* ntask, hostpart, ptid, MTU, NDF */
	char *p;
	fd_set wrk_rfds, wrk_wfds, rfds, wfds;
	int nfds;
	struct timeval tout;
	int *socks;
	int	dsock;					/* pvmd socket */
	struct frag *frpvmd = 0;	/* frag from pvmd */
	struct frag *topvmd = 0;	/* frag to pvmd */
	struct frag *frtask = 0;	/* frag from task */
	struct frag *totask;		/* frag being sent to task */
	struct frag *fp;
	char *txcp = 0;             /* point to remainder of topvmd */
    int txtogo = 0;             /* len of remainder of topvmd */
	int frtogo;					/* len of remainder of a fragment */
	int topvmd_dst;				/* dst of fragment being sent to pvmd */ 
	int topvmd_src;				/* src of fragment being sent to pvmd */
	int topvmd_ff;				/* ff of fragment being sent to pvmd */
	int	dst;					/* dst of fragment being sent to node */
	CMMD_mcb rmid;				/* msg IDs returned by async recv */
	int n;
	int typepart = TIDONNODE;
	char *cp;
	short *mc_dsts;				/* array to store multicast targets */
	int mc_ndst;				/* how many of them */


	pvmbeatask();

	CMMD_enable();

	pvm_getfds(&socks);
	dsock = socks[0];
	nfds = dsock + 1;

	if (p = getenv("PVMENTASK")) {
		n = atoi(p);
		pvminfo[0] = (n & tidnmask) + 1;	/* number of tasks to spawn */
		typepart += n & tidtmask;
	} else
		pvminfo[0] = CMMD_partition_size();

/*
sprintf(etext, "pvmhost: type = %d", typepart);
pvmlogerror(etext);
*/

	pvminfo[1] = (pvmmytid & pvmtidhmask) + typepart;
	pvminfo[2] = pvmmyptid;
	pvminfo[3] = MAXFRAGSIZE;
	pvminfo[4] = pvmmyndf;
	CMMD_bc_from_host(pvminfo, sizeof(pvminfo));

	frtask = fr_new(MAXFRAGSIZE);
	if ((rmid = ASYNCRECV(frtask->fr_dat, frtask->fr_max)) < 0) {
		pvmlogperror("pvmhost: ASYNCRECV");
		pvm_exit();
		exit(0);
	}

	FD_ZERO(&wrk_rfds);
	FD_ZERO(&wrk_wfds);
	FD_SET(dsock, &wrk_rfds);

	while (1) {			/* ferry messages between pvmd and tasks */
		
		tout.tv_sec = 0;
		tout.tv_usec = 0;
		if (!topvmd && ASYNCDONE(rmid)) {
			topvmd = frtask;
			txtogo = CMMD_mcb_bytes(rmid);		/* including header */
/*
sprintf(etext, "pvmhost: recv %d", txtogo);
pvmlogerror(etext);
*/
			frtask = fr_new(MAXFRAGSIZE);
			/* ready for the next message */
			CMMD_free_mcb(rmid);
			rmid = ASYNCRECV(frtask->fr_dat, frtask->fr_max);
			FD_SET(dsock, &wrk_wfds);
			txcp = topvmd->fr_dat;
			topvmd_dst = pvmget32(txcp);
			topvmd_src = pvmget32(txcp + 4);
			topvmd_ff = pvmget8(txcp + 12);
			if (txtogo > pvmudpmtu) {
				frtogo = pvmudpmtu;
				pvmput32(txcp + 8, frtogo - TDFRAGHDR);
				pvmput8(txcp + 12, (topvmd_ff & FFSOM));
			} else
				frtogo = txtogo;
		}
		rfds = wrk_rfds;
		wfds = wrk_wfds;

		if (select(nfds, &rfds, &wfds, (fd_set*)0, &tout) == -1 
		&& errno != EINTR) {
			pvmlogperror("pvmhost: select");
			exit(0);
		}

		if (FD_ISSET(dsock, &rfds)) {

			if (!frpvmd)
				frpvmd = fr_new(MAXFRAGSIZE);
			n = read(dsock, frpvmd->fr_dat + frpvmd->fr_len, frpvmd->fr_max -
                    (frpvmd->fr_dat - frpvmd->fr_buf) - frpvmd->fr_len);

/*
sprintf(etext, "pvmhost: read %d", n);
pvmlogerror(etext);
*/

			if (n == -1 && errno != EWOULDBLOCK && errno != EINTR) {
				pvmlogperror("pvmhost: read pvmd sock");
				exit(0);
			}
			if (!n) {
				pvmlogerror("pvmhost: read EOF on pvmd sock\n");
				exit(0);
			}

			frpvmd->fr_len += n;

			do {
				fp = frpvmd;
				if (fp->fr_len < TDFRAGHDR)
					break;
				n = TDFRAGHDR + pvmget32(fp->fr_dat + 8);   /* header + body */
				if (fp->fr_len < n)
					break;
				if (fp->fr_len > n) {
					frpvmd = fr_new(MAXFRAGSIZE);
					BCOPY(fp->fr_dat + n, frpvmd->fr_dat, fp->fr_len - n);
					frpvmd->fr_len = fp->fr_len - n;
					fp->fr_len = n;
				} else
					frpvmd = 0;

				dst = pvmget32(fp->fr_dat);
				if (dst == pvmmytid) {
					/* multicast info (XXX must fit into one frag) */
					cp = fp->fr_dat + TDFRAGHDR;
					fp->fr_len -= TDFRAGHDR;
					mc_ndst = fp->fr_len/sizeof(short);
					mc_dsts = TALLOC(mc_ndst, short, "mcdsts");
					BCOPY(cp, mc_dsts, fp->fr_len);
				} else if (dst & TIDGID) {
					/* multicast message body */
					cm_mcast(fp, mc_dsts, mc_ndst, pvminfo[1]);
					PVM_FREE(mc_dsts);
				} else {
					if (CMMD_send_noblock(dst & tidnmask, PMTDN, fp->fr_dat,
					fp->fr_len) == CMMD_ERRVAL) {
						sprintf(etext, "pvmhost: can't send to t%x", dst);
						pvmlogperror(etext);
					}
				}
				fr_unref(fp);

/*
sprintf(etext, "pvmhost: sent %d to node %d\n", fp->fr_len, (dst & tidnmask));
pvmlogerror(etext);
*/
				
			} while (frpvmd);
		}

		if (FD_ISSET(dsock, &wfds)) {
			n = write(dsock, txcp, frtogo);

/*
sprintf(etext, "pvmhost: wrote %d to pvmd\n", n);
pvmlogerror(etext);
*/

			if (n == -1 && errno != EWOULDBLOCK && errno != EINTR) {
				pvmlogperror("pvmhost: write pvmd sock");
				exit(0);
			}
			if (n > 0 && (txtogo -= n) > 0) {
				txcp += n;
				if (!(frtogo -= n)) {
					int ff = 0;

				/* new fragment */
					txcp -= TDFRAGHDR;
					txtogo += TDFRAGHDR;
					if (txtogo > pvmudpmtu)
						frtogo = pvmudpmtu;
					else {
						frtogo = txtogo;
						ff = topvmd_ff & FFEOM;
					}
					pvmput32(txcp, topvmd_dst);
					pvmput32(txcp + 4, topvmd_src);
					pvmput32(txcp + 8, frtogo - TDFRAGHDR);
					pvmput8(txcp + 12, ff);
				}
			}
			if (!txtogo) {		/* entire message sent */
				FD_CLR(dsock, &wrk_wfds);
				fr_unref(topvmd);
				topvmd = 0;
			}
		}
		CMMD_poll_for_services();
	}
}


/* 
 * make ntask copies of multicast fragment and send them to the nodes
 * listed in tids[]
 */
cm_mcast(fp, nodes, ntask, tidpart)
	struct frag *fp;	/* fragment to send */
	short nodes[];		/* targets */
	int ntask;			/* how many of them */
	int tidpart;		/* hostpart + typepart */
{
	int i;
	char *cp = fp->fr_dat;
	int dst;
	
	for (i = 0; i < ntask; i++) {
		dst = tidpart + nodes[i];
		pvmput32(cp, dst);
		if (CMMD_send_noblock(nodes[i], PMTDN, fp->fr_dat,
		fp->fr_len) == CMMD_ERRVAL) {
			sprintf(etext, "pvmhost: can't send to t%x", dst);
			pvmlogperror(etext);
		}
	}
}
