modules/ud/ud_process_stream.c
/* [<][>][^][v][top][bottom][index][help] */
FUNCTIONS
This source file includes following functions.
- get_NRTM_stream
- split_attribute
- reorder_attributes
- each_attribute_print
- print_object
- escape_apostrophes
- line_type
- report_transaction
- process_nrtm
- process_updates
- process_transaction
- UD_process_stream
1 /***************************************
2 $Revision: 1.24 $
3
4 Functions to process data stream( file, network socket, etc.)
5
6 Status: NOT REVUED, NOT TESTED
7
8 Author(s): Chris Ottrey, Andrei Robachevsky
9
10 ******************/ /******************
11 Modification History:
12 andrei (17/01/2000) Created.
13 ******************/ /******************
14 Copyright (c) 2000 RIPE NCC
15
16 All Rights Reserved
17
18 Permission to use, copy, modify, and distribute this software and its
19 documentation for any purpose and without fee is hereby granted,
20 provided that the above copyright notice appear in all copies and that
21 both that copyright notice and this permission notice appear in
22 supporting documentation, and that the name of the author not be
23 used in advertising or publicity pertaining to distribution of the
24 software without specific, written prior permission.
25
26 THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING
27 ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS; IN NO EVENT SHALL
28 AUTHOR BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
29 DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
30 AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
31 OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
32 ***************************************/
33 #include <sys/types.h>
34 #include <sys/socket.h>
35 #include <netdb.h>
36 #include <arpa/inet.h>
37 #include <unistd.h>
38 #include <sys/stat.h>
39 #include <fcntl.h>
40 #include <string.h>
41 #include "constants.h"
42 #include "query_command.h"
43 #include "ud.h"
44 #include "ud_int.h"
45
46 typedef enum _Line_Type_t {
47 LINE_ATTRIBUTE,
48 LINE_COMMENT,
49 LINE_EMPTY,
50 LINE_EOF,
51 LINE_ADD,
52 LINE_UPD,
53 LINE_DEL,
54 LINE_OVERRIDE_ADD,
55 LINE_OVERRIDE_UPD,
56 LINE_OVERRIDE_DEL,
57 LINE_PLUS
58 } Line_Type_t;
59
60 /* Maximum number of objects(serials) we can consume at a time */
61 #define SBUNCH 1000
62
63 static int report_transaction(Transaction_t *tr, Log_t *log, char *obj_name, char *reason);
64 static int process_nrtm(UD_stream_t *ud_stream, Transaction_t *tr, char *object_name, int operation);
65 static int process_updates(UD_stream_t *ud_stream, Transaction_t *tr, char *object_name, int operation);
66 static int process_transaction(UD_stream_t *ud_stream,Object_t *obj,char *object_name,nic_handle_t *nh, int operation);
67
68
69
70
71 /* temporary files to download serials */
72 char tmpfile1[STR_S], tmpfile2[STR_S];
73
74 /************************************************************
75 * FILE *get_NRTM_stream() *
76 * *
77 * Gets the NRTM stream *
78 * *
79 * First tries to request the serials from the NRTM server *
80 * If the name of the server appears to be not a network name*
81 * it tries to open the file with this name *
82 * *
83 * After downloading (opening) the serials into the file *
84 * it runs ripe2rpsl script to translate objects into RPSL *
85 * They are stored in a temporary file *
86 * *
87 * nrtm - pointer to _nrtm structure *
88 * upto_last - if==1 then requests to download serials using *
89 * LAST keyword *
90 * *
91 * Returns: *
92 * A pointer to the temp. file where serials in RPSL format *
93 * are stored *
94 * NULL - error *
95 * *
96 ************************************************************/
97 FILE *get_NRTM_stream(struct _nrtm *nrtm, int upto_last)
/* [<][>][^][v][top][bottom][index][help] */
98 {
99 int sockfd;
100 struct hostent *hptr;
101 struct sockaddr_in serv_addr;
102 struct in_addr *paddr;
103 char line_buff[STR_XXL];
104 FILE *fp;
105 int fd;
106 int fdtmp;
107 int nread, nwrite;
108 struct hostent result;
109 int error;
110 int network;
111
112
113 fprintf(stderr, "Making connection to NRTM server ...\n");
114 if ((sockfd=socket(AF_INET, SOCK_STREAM, 0))==-1){
115 perror("socket");
116 return(NULL);
117 }
118 /* hptr=gethostbyname(nrtm->server);*/
119 hptr=gethostbyname_r(nrtm->server, &result, line_buff, sizeof(line_buff), &error);
120 if (hptr) { /* this is a network stream*/
121 paddr=(struct in_addr *)hptr->h_addr;
122 bzero(&serv_addr, sizeof(serv_addr));
123 serv_addr.sin_family=AF_INET;
124 serv_addr.sin_port=nrtm->port;
125 memcpy(&serv_addr.sin_addr, paddr, sizeof(struct in_addr));
126 fprintf(stderr,"Trying %s port %d\n", inet_ntoa(serv_addr.sin_addr), nrtm->port);
127 if(connect(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr))==-1) {
128 perror("connect");
129 return(NULL);
130 }
131 fprintf(stderr, "Sending Invitation\n");
132
133 /* Request all available serials (upto LAST), or SBUNCH of them */
134 if(upto_last)
135 sprintf(line_buff, "-g RIPE:%d:%ld-LAST\n", nrtm->version, nrtm->current_serial+1);
136 else
137 sprintf(line_buff, "-g RIPE:%d:%ld-%ld\n", nrtm->version, nrtm->current_serial+1, nrtm->current_serial+SBUNCH);
138 nwrite=SK_write(sockfd, line_buff, strlen(line_buff) );
139 if(nwrite != strlen(line_buff)) { perror("write"); return(NULL); }
140 fd=sockfd;
141 network=1;
142 fprintf(stderr, "Returning stream pointer\n");
143 }
144 else { /* this is a file stream*/
145 network=0;
146 close(sockfd);
147 fprintf(stderr, "Trying file ...\n");
148 if((fd=open(nrtm->server, O_RDONLY, 0666))==-1) {
149 perror("open");
150 return(NULL);
151 }
152 }
153
154 /* make temporary files */
155 sprintf(tmpfile1, "temp.XXXX");
156
157 if((fdtmp=mkstemp(tmpfile1))==-1) {
158 perror("mkstemp");
159 close(fd);
160 return(NULL);
161 }
162
163 if(network) {
164 while ((nread=SK_read(fd, line_buff, sizeof(line_buff)))) {
165 if(nread==-1) return(NULL);
166 nwrite=write(fdtmp, line_buff,nread);
167 if(nread != nwrite) return(NULL);
168 }
169 } else {
170 while ((nread=read(fd, line_buff, sizeof(line_buff)))) {
171 if(nread==-1) return(NULL);
172 nwrite=write(fdtmp, line_buff,nread);
173 if(nread != nwrite) return(NULL);
174 }
175 }
176 close(fd); close(fdtmp);
177
178 sprintf(tmpfile2, "%s.2", tmpfile1);
179
180 sprintf(line_buff, "cat %s | ./ripe2rpsl > %s", tmpfile1, tmpfile2);
181 if (system(line_buff)!=0) return(NULL);
182 if((fp=fopen(tmpfile2, "r+"))==NULL)return(NULL);
183
184 unlink(tmpfile1);
185
186 return(fp);
187
188 }
189
190
191 /* XXX this is from qc module */
192 char *strsep(char **stringp, const char *delim);
193 /*
194 {
195 return strtok_r( *stringp, delim, stringp);
196 }
197
198 */
199
200 /* Delimiters that separate list members, both RPS(,) and legacy( ) */
201 #define ATTR_DELIMITERS " ,"
202
203 static GSList *split_attribute(GSList *attr_list, A_Type_t attr_type, char *attr_value){
/* [<][>][^][v][top][bottom][index][help] */
204 char *token;
205 char *split;
206 char *value, *n;
207 Attribute_t *attr_split;
208 GSList *the_list = attr_list;
209
210 /* check for end-of-line comments */
211 n = index(attr_value, '#');
212 /* if there is no comment check for trailing \n */
213 if(n == NULL) n = index(attr_value, '\n');
214 /* now copy the clean value into the attribute */
215 if(n == NULL) value = g_strdup(attr_value);
216 else value = g_strndup(attr_value, (n - attr_value));
217
218 token=value;
219 while((split=strsep(&token, ATTR_DELIMITERS))){
220 attr_split = attribute_new1(attr_type, split);
221 if (attr_split) the_list = g_slist_append(the_list, attr_split);
222 }
223 free(value);
224 return(the_list);
225 }
226
227 /************************************************************
228 * *
229 * The function to reorder attributes in the List *
230 * nic-hdl and mnt-by should come first *
231 * *
232 * should return 0 if they are equal, a negative value if *
233 * the first element comes before the second, or a positive *
234 * value if the first element comes after the second *
235 * *
236 ************************************************************/
237 static gint reorder_attributes(const void *element1, const void *element2)
/* [<][>][^][v][top][bottom][index][help] */
238 {
239 Attribute_t *attr1 = (Attribute_t *)element1;
240 Attribute_t *attr2 = (Attribute_t *)element2;
241 gint order = -1;
242
243 if(attr2->type == A_MB) order= 1;
244 if(attr1->type == A_MB) order= -1;
245 if(attr2->type == A_NH) order= 1;
246 if(attr1->type == A_NH) order= -1;
247
248 return(order);
249
250 }
251
252 /* XXX */
253 static void each_attribute_print(void *element_data, void *tr_ptr)
/* [<][>][^][v][top][bottom][index][help] */
254 {
255
256 Attribute_t *attr = (Attribute_t *)element_data;
257
258 fprintf(stderr, "[%d|%s]\n", attr->type, attr->value);
259
260 }
261
262 /* XXX */
263 static void print_object(Object_t *obj)
/* [<][>][^][v][top][bottom][index][help] */
264 {
265 g_slist_foreach(obj->attributes, each_attribute_print, NULL);
266 fprintf(stderr, ">>>>>\n%s\n", obj->object->str);
267 }
268
269
270 /******************************************************************
271 * GString *escape_apostrophes() *
272 * Escapes apostrophes in the text so they do not confuse printf *
273 * functions and don't corrupt SQL queries *
274 * *
275 * *****************************************************************/
276 GString *escape_apostrophes(GString *text) {
/* [<][>][^][v][top][bottom][index][help] */
277 int i;
278 for (i=0; i < text->len; i++) {
279 if ((text->str[i] == '\'') || (text->str[i] == '\\')) {
280 text = g_string_insert_c(text, i, '\\');
281 i++;
282 }
283 }
284 return(text);
285 } /* escape_apostrophes() */
286
287
288 /******************************************************************
289 * Line_Type_t line_type(e) *
290 * Determines the line type analysing the first letters *
291 * *
292 * ****************************************************************/
293 static Line_Type_t line_type(const char *line) {
/* [<][>][^][v][top][bottom][index][help] */
294 Line_Type_t result = -1;
295
296 if (strncmp(line, "# EOF", 4) == 0) {
297 result = LINE_EOF;
298 }
299 else if (strncmp(line, "#", 1) == 0) {
300 result = LINE_COMMENT;
301 }
302 else if (strcmp(line, "\n") == 0) {
303 result = LINE_EMPTY;
304 }
305 else if (strcmp(line, "ADD\n") == 0) {
306 result = LINE_ADD;
307 }
308 else if (strcmp(line, "UPD\n") == 0) {
309 result = LINE_UPD;
310 }
311 else if (strcmp(line, "DEL\n") == 0) {
312 result = LINE_DEL;
313 }
314 else if (strcmp(line, "ADD_OVERRIDE\n") == 0) {
315 result = LINE_OVERRIDE_ADD;
316 }
317 else if (strcmp(line, "UPD_OVERRIDE\n") == 0) {
318 result = LINE_OVERRIDE_UPD;
319 }
320 else if (strcmp(line, "DEL_OVERRIDE\n") == 0) {
321 result = LINE_OVERRIDE_DEL;
322 }
323 else if (strncmp(line, "+", 1) == 0) {
324 result = LINE_PLUS;
325 }
326 else {
327 result = LINE_ATTRIBUTE;
328 }
329
330 return result;
331 } /* line_type() */
332
333
334 /******************************************************************
335 * report_transaction() *
336 * *
337 * Prints error report to the log *
338 * *
339 * reason - additional message that will be included *
340 * *
341 * *****************************************************************/
342 static int report_transaction(Transaction_t *tr, Log_t *log, char *obj_name, char *reason)
/* [<][>][^][v][top][bottom][index][help] */
343 {
344 int result=0;
345
346 if(tr->succeeded==0) {
347 result=tr->error;
348 log->num_failed++;
349 fprintf(stderr, "FAILED[%s][%s(%d)](%d/%d)\n ", obj_name, reason, result, log->num_failed, (log->num_failed)+(log->num_ok));
350 fprintf(log->logfile, "*FAILED[%s][%s](%d/%d)\n ", obj_name, reason, log->num_failed, (log->num_failed)+(log->num_ok));
351 if(result & ERROR_U_MEM) fprintf(log->logfile, "\t*Memory allocation error\n");
352 if(result & ERROR_U_DBS) fprintf(log->logfile, "\t*Database (SQL) error\n");
353 if(result & ERROR_U_OBJ) fprintf(log->logfile, "\t*Object (RF) error\n");
354 if(result & ERROR_U_AUT) fprintf(log->logfile, "\t*Object authentication error\n");
355 if(result & ERROR_U_BADOP) fprintf(log->logfile, "\t*Bad operation\n");
356 if(result & ERROR_U_COP) fprintf(log->logfile, "\t*Conflicting operation\n");
357 if(result & ERROR_U_NSUP) fprintf(log->logfile, "\t*Object of this type is not supported\n");
358 if(result & ERROR_U_BUG) fprintf(log->logfile, "\t*Software bug - report to <ripe-dbm@ripe.net>\n");
359 fprintf(log->logfile, "%s", (tr->error_script)->str);
360 result=(-1)*result;
361 fflush(log->logfile);
362 }
363 else {
364 result=1;
365 log->num_ok++;
366 fprintf(stderr, "OK(%d/%d)\n", log->num_ok, (log->num_failed)+(log->num_ok));
367 }
368
369 return(result);
370 }/* report_transaction() */
371
372
373
374 /************************************************************
375 * process_nrtm() *
376 * *
377 * Process object in NRTM client mode *
378 * *
379 * nrtm - pointer to _nrtm structure *
380 * log - pointer to Log_t structure *
381 * object_name - name of the object *
382 * operation - operation code (OP_ADD/OP_DEL) *
383 * *
384 * Returns: *
385 * 1 - okay *
386 * <0 - error *
387 * *
388 ************************************************************/
389
390 static int process_nrtm(UD_stream_t *ud_stream, Transaction_t *tr, char *object_name, int operation)
/* [<][>][^][v][top][bottom][index][help] */
391 {
392 int result=0;
393 int dummy=0;
394 struct _nrtm *nrtm = ud_stream->nrtm;
395 Log_t *log_ptr= &(ud_stream->log);
396
397 switch (operation) {
398
399 case OP_ADD:
400 /* XXX This hack is made to allow NRTM updates for some inconsistent objects */
401 /* One of the examples is reference by name which looks like nic-handle */
402 /* For this purpose we allow dummy creation when updating an object */
403 tr->dummy=1;
404 if(nrtm->tr){ /*saved?*/
405 if(tr->object_id==0) {
406 /* object does not exist in the DB */
407 object_process(nrtm->tr); /* delete the previous(saved) object*/
408 result=report_transaction(nrtm->tr, log_ptr, nrtm->object_name, "NRTM:DEL:While deleting previous(saved)");
409 if(result==1)create_serial(nrtm->tr);
410 object_free(nrtm->tr->object);
411 transaction_free(nrtm->tr); nrtm->tr=NULL;
412 /* Create an object and update NHR */
413 tr->action=(TA_CREATE | TA_UPD_NHR);
414 /* fprintf(stderr,"CREATE next\n"); */
415 object_process(tr); /* create a new one*/
416 result=report_transaction(tr, log_ptr, object_name, "NRTM:ADD:While creating new");
417 if(result==1)create_serial(tr);
418 }
419 else {
420 /* object already exists in the DB - update or dummy replacement*/
421 if(tr->object_id==nrtm->tr->object_id) {/*compare the two, may be we may collapse operations*/
422 object_free(nrtm->tr->object);
423 transaction_free(nrtm->tr); nrtm->tr=NULL;
424 /* fprintf(stderr,"DEL-ADD ->> UPDATE\n");*/
425 tr->action=TA_UPDATE;
426 object_process(tr);
427 report_transaction(tr, log_ptr, object_name,"NRTM:upd");
428 result=report_transaction(tr, log_ptr, object_name,"NRTM:upd");
429 if(result==1)create_serial(tr);
430 }
431 else { /* this should be a dummy object in the database(that we are going to replace with the real one */
432 /* or an interleaved operation*/
433 /* fprintf(stderr,"DEL previous\n");*/
434 object_process(nrtm->tr); /* delete the previous(saved) object*/
435 result=report_transaction(nrtm->tr, log_ptr, nrtm->object_name, "NRTM:DEL:While deleting previous(saved)");
436 if(result==1)create_serial(nrtm->tr);
437 object_free(nrtm->tr->object);
438 transaction_free(nrtm->tr); nrtm->tr=NULL;
439 tr->action=TA_UPDATE;
440 dummy=isdummy(tr);
441 /* If we are replacing dummy with a real object update NHR */
442 if(dummy==1) tr->action |= TA_UPD_NHR;
443 /* fprintf(stderr,"UPDATE next(dummy)\n"); */
444 object_process(tr); /* create a new one*/
445 result=report_transaction(tr, log_ptr, object_name, "NRTM:ADD:While creating new");
446 /* check if it was dummy */
447 if (dummy==1)
448 tr->action=TA_CREATE; /* we don't want to generate DEL serial for dummy replacement*/
449 if(result==1)create_serial(tr);
450 }
451 }
452 }
453 else { /* brand new object*/
454 if(tr->object_id==0) {
455 /* fprintf(stderr,"CREATE new\n");*/
456 /* Create an object and update NHR */
457 tr->action=(TA_CREATE | TA_UPD_NHR);
458 object_process(tr);
459 result=report_transaction(tr, log_ptr, object_name,"NRTM:ADD:While creating new");
460 if(result==1)create_serial(tr);
461 }
462 else { /* this may happen because of dummies*/
463 /* fprintf(stderr,"CREATE new\n");*/
464 tr->action=TA_UPDATE;
465 dummy=isdummy(tr);
466 /* If we are replacing dummy with a real object update NHR */
467 if(dummy==1) tr->action |= TA_UPD_NHR;
468 object_process(tr);
469 result=report_transaction(tr, log_ptr, object_name,"NRTM:ADD:While creating new");
470 /* check if it was dummy */
471 if (dummy==1)
472 tr->action=TA_CREATE; /* we don't want to generate DEL serial for dummy replacement*/
473 if(result==1)create_serial(tr);
474 }
475 }
476 break;
477
478 case OP_DEL:
479 if(nrtm->tr){ /*saved?*/
480 /* fprintf(stderr,"DEL previous\n");*/
481 object_process(nrtm->tr); /* delete the previous(saved) object*/
482 result=report_transaction(nrtm->tr, log_ptr, nrtm->object_name, "NRTM:DEL:While deleting previous(saved) object");
483 if(result==1)create_serial(nrtm->tr);
484 object_free(nrtm->tr->object);
485 transaction_free(nrtm->tr); nrtm->tr=NULL;
486 }
487 if(tr->object_id>0){ /* save the object*/
488 fprintf(stderr,"SAVED\n");
489 tr->action=TA_DELETE;
490 nrtm->tr=tr;
491 strcpy(nrtm->object_name, object_name);
492 return(1);
493 }
494 else { /* this is an error*/
495 tr->succeeded=0; tr->error|=ERROR_U_COP;
496 result=report_transaction(tr, log_ptr, object_name, "NRTM:OOS:Trying to DEL non-existing object");
497 /* fprintf(stderr,"Lost sync. Skipping\n");*/
498 }
499 break;
500
501 default:
502 tr->succeeded=0; tr->error |=ERROR_U_BADOP;
503 break;
504 }
505
506 /* Free resources */
507 object_free(tr->object);
508 transaction_free(tr);
509
510 return(result);
511 } /* process_nrtm() */
512
513
514
515 /************************************************************
516 * process_updates() *
517 * *
518 * Process object in update mode *
519 * *
520 * ud_stream - pointer to UD_stream structure *
521 * object_name - name of the object *
522 * operation - operation code (OP_ADD/OP_DEL) *
523 * *
524 * Note: *
525 * Frees tr and tr->obj on exit *
526 * *
527 * Returns: *
528 * 1 - okay *
529 * <0 - error *
530 * *
531 ************************************************************/
532
533 static int process_updates(UD_stream_t *ud_stream, Transaction_t *tr, char *object_name, int operation)
/* [<][>][^][v][top][bottom][index][help] */
534 {
535 int result=0;
536 Log_t *log_ptr= &(ud_stream->log);
537 int dummy=0;
538
539 switch(operation) {
540 /* Compare operations and report an error if they do not match */
541 case OP_ADD:
542 if(tr->object_id!=0) { /* trying to create, but object exists */
543 tr->succeeded=0; tr->error|=ERROR_U_COP;
544 } else {
545 /* Action: create the object and update NHR */
546 tr->action=(TA_CREATE | TA_UPD_NHR);
547 object_process(tr);
548 }
549 break;
550 case OP_UPD:
551 if(tr->object_id==0) { /* trying to update non-existing object*/
552 tr->succeeded=0; tr->error|=ERROR_U_COP;
553 } else {
554 tr->action=TA_UPDATE;
555 dummy=isdummy(tr);
556 /* If we are replacing dummy with a real object update NHR */
557 if(dummy==1) tr->action |= TA_UPD_NHR;
558 object_process(tr);
559 }
560 break;
561
562 case OP_DEL:
563 if(tr->object_id==0) { /* trying t delete non-existing object*/
564 tr->succeeded=0; tr->error|=ERROR_U_COP;
565 } else {
566 tr->action=TA_DELETE;
567 object_process(tr);
568 }
569 break;
570
571 default:
572 /* bad operation for this mode if not standalone */
573 if(tr->standalone) {
574 if(tr->object_id==0)tr->action=TA_CREATE; else tr->action=TA_UPDATE;
575 object_process(tr);
576 }
577 else {
578 tr->succeeded=0;
579 tr->error|=ERROR_U_BADOP;
580 }
581 break;
582 }
583 /* Make a report */
584 result=report_transaction(tr, log_ptr, object_name, "RIPupd:");
585
586 /* If not in standalone mode create serial and copy error transcript */
587 if(!tr->standalone) {
588 if(result==1)create_serial(tr);
589 ud_stream->error_script=g_strdup((tr->error_script)->str);
590 }
591
592 /* Free resources */
593 object_free(tr->object);
594 transaction_free(tr);
595
596 return(result);
597
598 } /* process_updates() */
599
600
601 /************************************************************
602 * *
603 * int process_transaction() *
604 * *
605 * Processes the transaction *
606 * *
607 * ud_stream - pointer to UD_stream_t structure *
608 * *
609 * Returns: *
610 * 1 - no error *
611 * <0- errors *
612 * *
613 ************************************************************/
614
615 /* It frees the obj */
616
617 static int process_transaction(UD_stream_t *ud_stream,
/* [<][>][^][v][top][bottom][index][help] */
618 Object_t *obj,
619 char *object_name,
620 nic_handle_t *nh,
621 int operation)
622 {
623 Transaction_t *tr = NULL;
624 Log_t *log_ptr = &(ud_stream->log);
625 Attribute_t *attr=NULL;
626 int result;
627
628 /* start new transaction now */
629 tr = transaction_new(ud_stream->db_connection, obj->type);
630
631 /* Return with error if transaction cannot be created */
632 if (tr == NULL) return(-1);
633
634 tr->standalone=IS_STANDALONE(ud_stream->ud_mode);
635 tr->dummy=IS_DUMMY_ALLOWED(ud_stream->ud_mode);
636 tr->load_pass=ud_stream->load_pass;
637 tr->object=obj;
638 tr->nh=nh;
639
640 /* We perform no commit/rollback in the loader mode, so thread_id should be set to 0 */
641 if(ud_stream->load_pass!=0) { tr->thread_ins=0; tr->thread_upd=0; }
642
643 /* For the first load pass we only create objects */
644 if(ud_stream->load_pass==1) tr->object_id=0;
645 else tr->object_id=get_object_id(tr);
646
647 /* Object cannot be retrieved */
648 if(tr->object_id==-1) { /* DB error*/
649 tr->succeeded=0;
650 tr->error |= ERROR_U_DBS;
651 report_transaction(tr, log_ptr, object_name, "Object cannot be retrieved");
652 transaction_free(tr);
653 object_free(obj);
654 return(-1);
655 }
656 /* save the name of person/role as we need it for referential */
657 /* integrity check when deleting the object against names. */
658 /* This is needed to support legacy references by name rather */
659 /* then by nic_hdl */
660 if((tr->class_type==C_PN) || (tr->class_type==C_RO)){
661 attr = attribute_new(object_name);
662 if (attr==NULL) {
663 tr->succeeded=0;
664 tr->error |= ERROR_U_MEM;
665 report_transaction(tr, log_ptr, object_name, "Cannot allocate memery");
666 transaction_free(tr);
667 object_free(obj);
668 return(-1);
669 }
670 /* Save the value */
671 tr->save=g_strdup(attr->value);
672 }
673
674 /* Process transaction. tr and obj are freed inside the process_* functions */
675
676 if(IS_UPDATE(ud_stream->ud_mode))
677 /* We are in update mode */
678 result=process_updates(ud_stream, tr, object_name, operation);
679 else
680 /* We are in NRTM mode */
681 result=process_nrtm(ud_stream, tr, object_name, operation);
682
683 /* free attr if has been allocated */
684 if(attr) attribute_free(attr, NULL);
685
686 return(result);
687
688 }
689
690
691 /************************************************************
692 * *
693 * int UD_process_stream(UD_stream_t *ud_stream) *
694 * *
695 * Processes the stream *
696 * *
697 * ud_stream - pointer to UD_stream_t structure *
698 * *
699 * Returns: *
700 * in update mode (!standalone)(1 object processed): *
701 * 1 - no error *
702 * <0- errors *
703 * *
704 * in NRTM & standalone modes *
705 * total number of object processed *
706 * *
707 ************************************************************/
708
709 int UD_process_stream(UD_stream_t *ud_stream)
/* [<][>][^][v][top][bottom][index][help] */
710 {
711 char line_buff[STR_XXL], object_name[STR_XXL];
712 GString *g_line_buff; // needed to escape apostrophes
713 GSList *class_attr_list = NULL;
714 Attribute_t *class_attr;
715 Attribute_t *attr;
716 nic_handle_t *nh_ptr=NULL; /* To save NIC handle structure */
717 Object_t *obj = NULL;
718 Transaction_t *tr = NULL;
719 SQ_connection_t *sql_connection;
720 int start_object;
721 int a_type;
722 char *a_value;
723 char *ptr;
724 /* here we will store the parsed nic-hdl in required format */
725 char nic[MAX_NH_LENGTH];
726 long num_skip;
727 struct _nrtm *nrtm;
728 Log_t *log_ptr= &(ud_stream->log);
729 time_t stime, ftime;
730 double obj_second1, obj_second10;
731 int result;
732 int operation=0;
733 int interrupt=0;
734 int do_update;
735 int default_ud_mode = ud_stream->ud_mode;
736 Line_Type_t linetype;
737
738 nrtm=ud_stream->nrtm;
739 start_object = 1;
740 a_type=-1;
741
742
743 /* Allocate line bufer */
744 if ((g_line_buff = g_string_sized_new(STR_XXL)) == NULL){
745 fprintf(stderr, "E: cannot allocate gstring\n");
746 return(-1);
747 }
748
749 /* Check connection to the database */
750 if(mysql_ping(ud_stream->db_connection)) {
751 fprintf(stderr, "D: ERROR: no SQL connection\n");
752 g_string_free(g_line_buff, TRUE);
753 return(-1);
754 }
755
756 fprintf(stderr, "OK\n");
757 sql_connection=ud_stream->db_connection;
758
759 /* This is useful for loading DB from huge disk file. */
760 /* We may start from <num_skip>th object */
761 num_skip=ud_stream->num_skip;
762 if(num_skip>0) fprintf(stderr, "skipping %lu records\n", num_skip);
763
764 /* Start timer for statistics */
765 stime=time(NULL);
766
767 /* Main loop. Reading input stream line by line */
768 /* Empty line signals to start processing an object, if we have it */
769 while (fgets(line_buff, STR_XXL, ud_stream->stream) != NULL) {
770
771 switch (linetype=line_type(line_buff)) {
772 case LINE_PLUS:
773 case LINE_ATTRIBUTE:
774 if (start_object == 1) {
775 /* This is for loading stuff */
776 /* if(num_skip>0){ fprintf(stderr, "\r%10lu", num_skip); num_skip--; log_ptr->num_ok++; break; } */
777 obj = object_new(line_buff);
778 }
779 if (obj) {
780 g_string_sprintf(g_line_buff, "%s", line_buff);
781 /* escape apostrophes in the input line */
782 g_line_buff=escape_apostrophes(g_line_buff);
783
784 if(start_object){
785 /* If this is the first attribute(==object name/type) */
786 start_object=0;
787 strncpy(object_name, g_line_buff->str, g_line_buff->len-1);
788 *(object_name+g_line_buff->len-1)='\0';
789 fprintf(stderr, "D: object: [%s] ", object_name);
790 /* Create an attribute - the first one determines a class */
791 class_attr_list=NULL; /* Initialize the list that will hold splitted class attribute */
792 class_attr = attribute_new(g_line_buff->str);
793 if (class_attr == NULL) die; /* Should not happen */
794 if((class_attr->type==A_PN)||(class_attr->type==A_RO)){
795 /* split names */
796 class_attr_list = split_attribute(class_attr_list, class_attr->type, class_attr->value);
797 attribute_free(class_attr, NULL);
798 } else {
799 class_attr_list = g_slist_append(class_attr_list, class_attr);
800 }
801 /* do nothing more with this attribute - we will prepend it at the end */
802 }
803 else {
804
805 attr = attribute_new(g_line_buff->str);
806
807 if (attr) {
808 a_type=attr->type;
809 a_value=attr->value;
810 if(a_type==A_NH) {
811 /* Parse the string into nh structure */
812 /* In case of an AUTO NIC handle check the ID in the database */
813 /* Possible errors leave to core processing */
814 if(NH_parse(attr->value, &nh_ptr) == 0) {
815 /* fprintf(stderr, "D:parsing NIC: [%s]\n", attr->value); */
816 /* Check if we can allocate it */
817 if((result = NH_check(nh_ptr, sql_connection))>0){
818 /* Convert nh to the database format */
819 NH_convert(nic, nh_ptr);
820 /* fprintf(stderr, "D:NIC:[%s]\n", nic); */
821 /* Replace NIC handle in the string which is copied to the text object */
822 sprintf(line_buff, g_line_buff->str);
823 ptr = strstr(line_buff, attr->value);
824 /* compose new attribute string */
825 strcpy(ptr, nic);
826 g_string_sprintf(g_line_buff, line_buff);
827 g_string_sprintfa(g_line_buff, "\n");
828 /* Update the attribute */
829 attribute_upd(attr, attr->type, nic);
830 /* fprintf(stderr, "D:attribute updated\n"); */
831 }
832 }
833 } /* NHR stuff */
834 }
835 else {
836 if(linetype==LINE_PLUS)a_value=g_line_buff->str+1; /* skip '+' sign */
837 else a_value=g_line_buff->str;
838 }
839 if (a_type>=0) { /* This indicates that the input line contains the value of the attribute */
840 switch (a_type) {
841 /*these attributes may appear several on the line - split them*/
842 case A_PN: /* person */
843 case A_RO: /* role */
844 case A_MR: /* mbrs-by-ref */
845 case A_MB: /* mnt-by */
846 case A_MO: /* member-of */
847 case A_SD: /* sub-dom */
848 case A_RZ: /* rev-srv */
849 case A_NS: /* nserver */
850 obj->attributes = split_attribute(obj->attributes, a_type, a_value);
851 if (attr) attribute_free(attr, NULL);
852 attr=NULL;
853 break;
854 default: break;
855 }
856 /* g_string_sprintfa(obj->object, "%s", g_line_buff->str); */
857 if(attr)obj->attributes = g_slist_append(obj->attributes, attr);
858 }
859 } /* if not start_object (not the first/class attribute) */
860 /* copy the line into object no matter whether it is an attribute or not (continualtion, etc.) */
861 g_string_sprintfa(obj->object, "%s", g_line_buff->str);
862 }/* if (obj) */
863 break;
864
865 case LINE_COMMENT:
866 break;
867
868 case LINE_EOF:
869 break;
870
871 case LINE_ADD:
872 /* restore the default operation mode */
873 operation=OP_ADD;
874 ud_stream->ud_mode=default_ud_mode;
875 break;
876
877 case LINE_OVERRIDE_ADD:
878 /* for override - switch the dummy bit on */
879 operation=OP_ADD;
880 ud_stream->ud_mode=default_ud_mode|B_DUMMY;
881 break;
882
883 case LINE_UPD:
884 /* restore the default operation mode */
885 operation=OP_UPD;
886 ud_stream->ud_mode=default_ud_mode;
887 break;
888
889 case LINE_OVERRIDE_UPD:
890 /* for override - switch the dummy bit on */
891 operation=OP_UPD;
892 ud_stream->ud_mode=default_ud_mode|B_DUMMY;
893 break;
894
895 case LINE_DEL:
896 /* restore the default operation mode */
897 operation=OP_DEL;
898 ud_stream->ud_mode=default_ud_mode;
899 break;
900
901 case LINE_OVERRIDE_DEL:
902 /* for override - switch the dummy bit on */
903 operation=OP_DEL;
904 ud_stream->ud_mode=default_ud_mode|B_DUMMY;
905 break;
906
907 case LINE_EMPTY:
908 /* Indicate that we have complete object, so a new one will be collected later */
909 start_object=1;
910 a_type=-1;
911 /* start processing the object */
912 if (obj == NULL) break; /* may be the previous lines were just garbage*/
913 /* reorder some attributes */
914 obj->attributes = g_slist_sort(obj->attributes, reorder_attributes);
915 /* prepend the class attribute */
916 obj->attributes = g_slist_concat(class_attr_list, obj->attributes);
917 /* forget the location */
918 class_attr=NULL;
919
920 /* XXX */
921 /* print_object(obj); */
922
923 /* start new transaction now */
924 result=process_transaction(ud_stream, obj, object_name, nh_ptr, operation);
925
926 /* process_transaction() frees tr and obj structures, */
927 /* so make sure we'll not reference these objects in the future */
928 obj=NULL; operation=OP_NOOP;
929 nh_ptr=NULL;
930 ud_stream->ud_mode=default_ud_mode;
931
932 /* this is a good place for quick interrupt */
933 do_update=CO_get_do_update();
934 if (do_update) interrupt=0; else interrupt=1;
935 /* we still need to exit in update server mode (only 1 object at a time */
936 if (IS_UPDATE(ud_stream->ud_mode) && (!IS_STANDALONE(ud_stream->ud_mode))) interrupt=1;
937
938 break;
939
940 default:
941 fprintf(stderr, "ERROR: Bad line type\n");
942 } /* switch */
943
944 /* Finish processing if interrupt has been set */
945 if (interrupt) break;
946 } /* while */
947
948 /* Some postprocessing */
949 if(!IS_UPDATE(ud_stream->ud_mode)){
950 /* We are in NRTM mode */
951 /* Clean up */
952 fclose(ud_stream->stream);
953 unlink(tmpfile2);
954 /* In NRTM mode there may be a saved object that is unprocessed */
955 if(nrtm->tr){ /*saved backlog?*/
956 object_process(nrtm->tr); /* delete the previous(saved) object*/
957 result=report_transaction(nrtm->tr, &(ud_stream->log), nrtm->object_name,
958 "NRTM:DEL:While deleting previous(saved) object");
959 if(result==1) create_serial(nrtm->tr);
960 object_free(nrtm->tr->object);
961 transaction_free(nrtm->tr); nrtm->tr=NULL;
962 }
963 }
964
965 /* That's all. Free GString */
966 g_string_free(g_line_buff, TRUE);
967
968 /* Calculate some statistics */
969 ftime=time(NULL);
970 obj_second1 = (float)(log_ptr->num_ok)/(ftime-stime);
971 obj_second10 = (float)(log_ptr->num_ok+log_ptr->num_failed)/(ftime-stime);
972
973 /* Print the report */
974 if(IS_STANDALONE(ud_stream->ud_mode) || (!IS_UPDATE(ud_stream->ud_mode))) {
975 /* printf("\n\n******** report **********\n%d objects OK\n%d objects failed\n",
976 log_ptr->num_ok, log_ptr->num_failed); */
977 fprintf(log_ptr->logfile,"\n******** report **********\n");
978 fprintf(log_ptr->logfile," %d objects OK (%5.2f obj/s)\n", log_ptr->num_ok, obj_second1);
979 fprintf(log_ptr->logfile," %d objects failed\n", log_ptr->num_failed);
980 fprintf(log_ptr->logfile," average processing time %5.2f obj/s (%5.2f obj/min)\n",
981 obj_second10, obj_second10*60);
982 result=log_ptr->num_ok+log_ptr->num_failed;
983 }
984 return(result);
985
986 } /* UD_process_stream */
987