#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "gen_tile.h" #include "protocol.h" #include "render_config.h" #include "dir_utils.h" #ifndef METATILE #warning("render_list not implemented for non-metatile mode. Feel free to submit fix") int main(int argc, char **argv) { fprintf(stderr, "render_list not implemented for non-metatile mode. Feel free to submit fix!\n"); return -1; } #else static int minZoom = 0; static int maxZoom = 18; static int verbose = 0; static int maxLoad = MAX_LOAD_OLD; int work_complete; void display_rate(struct timeval start, struct timeval end, int num) { int d_s, d_us; float sec; d_s = end.tv_sec - start.tv_sec; d_us = end.tv_usec - start.tv_usec; sec = d_s + d_us / 1000000.0; printf("Rendered %d tiles in %.2f seconds (%.2f tiles/s)\n", num, sec, num / sec); fflush(NULL); } static time_t getPlanetTime(char *tile_dir) { static time_t last_check; static time_t planet_timestamp; time_t now = time(NULL); struct stat buf; char filename[PATH_MAX]; snprintf(filename, PATH_MAX-1, "%s/%s", tile_dir, PLANET_TIMESTAMP); // Only check for updates periodically if (now < last_check + 300) return planet_timestamp; last_check = now; if (stat(filename, &buf)) { fprintf(stderr, "Planet timestamp file (%s) is missing\n", filename); // Make something up planet_timestamp = now - 3 * 24 * 60 * 60; } else { if (buf.st_mtime != planet_timestamp) { printf("Planet file updated at %s", ctime(&buf.st_mtime)); planet_timestamp = buf.st_mtime; } } return planet_timestamp; } int get_load_avg(void) { FILE *loadavg = fopen("/proc/loadavg", "r"); int avg = 1000; if (!loadavg) { fprintf(stderr, "failed to read /proc/loadavg"); return 1000; } if (fscanf(loadavg, "%d", &avg) != 1) { fprintf(stderr, "failed to parse /proc/loadavg"); fclose(loadavg); return 1000; } fclose(loadavg); return avg; } int process_loop(int fd, const char *mapname, int x, int y, int z) { struct protocol cmd, rsp; //struct pollfd fds[1]; int ret = 0; bzero(&cmd, sizeof(cmd)); cmd.ver = 2; cmd.cmd = cmdRenderBulk; cmd.z = z; cmd.x = x; cmd.y = y; strcpy(cmd.xmlname, mapname); //printf("Sending request\n"); ret = send(fd, &cmd, sizeof(cmd), 0); if (ret != sizeof(cmd)) { perror("send error"); } //printf("Waiting for response\n"); bzero(&rsp, sizeof(rsp)); ret = recv(fd, &rsp, sizeof(rsp), 0); if (ret != sizeof(rsp)) { perror("recv error"); return 0; } //printf("Got response\n"); if (rsp.cmd != cmdDone) { printf("rendering failed, pausing\n"); sleep(10); } if (!ret) perror("Socket send error"); return ret; } void process(int fd, const char *name) { char xmlconfig[XMLCONFIG_MAX]; int x, y, z; if (path_to_xyz(name, xmlconfig, &x, &y, &z)) return; printf("Requesting xml(%s) x(%d) y(%d) z(%d)\n", xmlconfig, x, y, z); process_loop(fd, xmlconfig, x, y, z); } static void check_load(void) { int avg = get_load_avg(); while (avg >= maxLoad) { printf("Load average %d, sleeping\n", avg); sleep(5); avg = get_load_avg(); } } #define QMAX 32 pthread_mutex_t qLock; pthread_cond_t qCondNotEmpty; pthread_cond_t qCondNotFull; unsigned int qLen; struct qItem { char *path; struct qItem *next; }; struct qItem *qHead, *qTail; char *fetch(void) { // Fetch path to render from queue or NULL on work completion // Must free() pointer after use char *path; pthread_mutex_lock(&qLock); while (qLen == 0) { if (work_complete) { pthread_mutex_unlock(&qLock); return NULL; } pthread_cond_wait(&qCondNotEmpty, &qLock); } // Fetch item from queue if (!qHead) { fprintf(stderr, "Queue failure, null qHead with %d items in list\n", qLen); exit(1); } path = qHead->path; if (qHead == qTail) { free(qHead); qHead = NULL; qTail = NULL; qLen = 0; } else { struct qItem *e = qHead; qHead = qHead->next; free(e); if (qLen == QMAX) pthread_cond_signal(&qCondNotFull); qLen--; } pthread_mutex_unlock(&qLock); return path; } void enqueue(const char *path) { // Add this path in the local render queue struct qItem *e = malloc(sizeof(struct qItem)); e->path = strdup(path); e->next = NULL; if (!e->path) { fprintf(stderr, "Malloc failure\n"); exit(1); } pthread_mutex_lock(&qLock); while (qLen == QMAX) { pthread_cond_wait(&qCondNotFull, &qLock); } // Append item to end of queue if (qTail) qTail->next = e; else qHead = e; qTail = e; pthread_cond_signal(&qCondNotEmpty); qLen++; pthread_mutex_unlock(&qLock); } void *thread_main(void *arg) { const char *spath = (const char *)arg; int fd; struct sockaddr_un addr; char *tile; fd = socket(PF_UNIX, SOCK_STREAM, 0); if (fd < 0) { fprintf(stderr, "failed to create unix socket\n"); exit(2); } bzero(&addr, sizeof(addr)); addr.sun_family = AF_UNIX; strncpy(addr.sun_path, spath, sizeof(addr.sun_path)); if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) < 0) { fprintf(stderr, "socket connect failed for: %s\n", spath); close(fd); return NULL; } while((tile = fetch())) { process(fd, tile); free(tile); } close(fd); return NULL; } pthread_t *workers; void spawn_workers(int num, const char *spath) { int i; // Setup request queue pthread_mutex_init(&qLock, NULL); pthread_cond_init(&qCondNotEmpty, NULL); pthread_cond_init(&qCondNotFull, NULL); printf("Starting %d rendering threads\n", num); workers = calloc(sizeof(pthread_t), num); if (!workers) { perror("Error allocating worker memory"); exit(1); } for(i=0; i 18) { fprintf(stderr, "Invalid minimum zoom selected, must be between 0 and 18\n"); return 1; } break; case 'Z': /* -Z, --max-zoom */ maxZoom=atoi(optarg); if (maxZoom < 0 || maxZoom > 18) { fprintf(stderr, "Invalid maximum zoom selected, must be between 0 and 18\n"); return 1; } break; case 'f': /* -f, --force */ force=1; break; case 'v': /* -v, --verbose */ verbose=1; break; case 'h': /* -h, --help */ fprintf(stderr, "Usage: render_list [OPTION] ...\n"); fprintf(stderr, " -a, --all render all tiles in given zoom level range instead of reading from STDIN\n"); fprintf(stderr, " -f, --force render tiles even if they seem current\n"); fprintf(stderr, " -m, --map=MAP render tiles in this map (defaults to '" XMLCONFIG_DEFAULT "')\n"); fprintf(stderr, " -l, --max-load=LOAD sleep if load is this high (defaults to %d)\n", MAX_LOAD_OLD); fprintf(stderr, " -s, --socket=SOCKET unix domain socket name for contacting renderd\n"); fprintf(stderr, " -n, --num-threads=N the number of parallel request threads (default 1)\n"); fprintf(stderr, " -t, --tile-dir tile cache directory (defaults to '" HASH_PATH "')\n"); fprintf(stderr, " -z, --min-zoom=ZOOM filter input to only render tiles greater or equal to this zoom level (default is 0)\n"); fprintf(stderr, " -Z, --max-zoom=ZOOM filter input to only render tiles less than or equal to this zoom level (default is 18)\n"); fprintf(stderr, "If you are using --all, you can restrict the tile range by adding these options:\n"); fprintf(stderr, " -x, --min-x=X minimum X tile coordinate\n"); fprintf(stderr, " -X, --max-x=X maximum X tile coordinate\n"); fprintf(stderr, " -y, --min-y=Y minimum Y tile coordinate\n"); fprintf(stderr, " -Y, --max-y=Y maximum Y tile coordinate\n"); fprintf(stderr, "Without --all, send a list of tiles to be rendered from STDIN in the format:\n"); fprintf(stderr, " X Y Z\n"); fprintf(stderr, "e.g.\n"); fprintf(stderr, " 0 0 1\n"); fprintf(stderr, " 0 1 1\n"); fprintf(stderr, " 1 0 1\n"); fprintf(stderr, " 1 1 1\n"); fprintf(stderr, "The above would cause all 4 tiles at zoom 1 to be rendered\n"); return -1; default: fprintf(stderr, "unhandled char '%c'\n", c); break; } } if (maxZoom < minZoom) { fprintf(stderr, "Invalid zoom range, max zoom must be greater or equal to minimum zoom\n"); return 1; } if (all) { if ((minX != -1 || minY != -1 || maxX != -1 || maxY != -1) && minZoom != maxZoom) { fprintf(stderr, "min-zoom must be equal to max-zoom when using min-x, max-x, min-y, or max-y options\n"); return 1; } if (minX == -1) { minX = 0; } if (minY == -1) { minY = 0; } int lz = (1 << minZoom) - 1; if (minZoom == maxZoom) { if (maxX == -1) { maxX = lz; } if (maxY == -1) { maxY = lz; } if (minX > lz || minY > lz || maxX > lz || maxY > lz) { fprintf(stderr, "Invalid range, x and y values must be <= %d (2^zoom-1)\n", lz); return 1; } } if (minX < 0 || minY < 0 || maxX < -1 || maxY < -1) { fprintf(stderr, "Invalid range, x and y values must be >= 0\n"); return 1; } } fprintf(stderr, "Rendering client\n"); planetTime = getPlanetTime(tile_dir); gettimeofday(&start, NULL); spawn_workers(numThreads, spath); if (all) { int x, y, z; printf("Rendering all tiles from zoom %d to zoom %d\n", minZoom, maxZoom); for (z=minZoom; z <= maxZoom; z++) { int current_maxX = (maxX == -1) ? (1 << z)-1 : maxX; int current_maxY = (maxY == -1) ? (1 << z)-1 : maxY; printf("Rendering all tiles for zoom %d from (%d, %d) to (%d, %d)\n", z, minX, minY, current_maxX, current_maxY); for (x=minX; x <= current_maxX; x+=METATILE) { for (y=minY; y <= current_maxY; y+=METATILE) { check_load(); xyz_to_meta(name, sizeof(name), tile_dir, mapname, x, y, z); enqueue(name); //process_loop(fd, mapname, x, y, z); num_all++; num_render++; } } } } else { while(!feof(stdin)) { struct stat s; int n = fscanf(stdin, "%d %d %d", &x, &y, &z); if (n != 3) { // Discard input line char tmp[1024]; char *r = fgets(tmp, sizeof(tmp), stdin); if (!r) continue; // fprintf(stderr, "bad line %d: %s", num_all, tmp); continue; } if (z < minZoom || z > maxZoom) continue; printf("got: x(%d) y(%d) z(%d)\n", x, y, z); check_load(); num_all++; xyz_to_meta(name, sizeof(name), tile_dir, mapname, x, y, z); if (force || (stat(name, &s) < 0) || (planetTime > s.st_mtime)) { // missing or old, render it //ret = process_loop(fd, mapname, x, y, z); enqueue(name); num_render++; if (!(num_render % 10)) { gettimeofday(&end, NULL); printf("\n"); printf("Meta tiles rendered: "); display_rate(start, end, num_render); printf("Total tiles rendered: "); display_rate(start, end, num_render * METATILE * METATILE); printf("Total tiles handled from input: "); display_rate(start, end, num_all); } } } } finish_workers(numThreads); gettimeofday(&end, NULL); printf("\nTotal for all tiles rendered\n"); printf("Meta tiles rendered: "); display_rate(start, end, num_render); printf("Total tiles rendered: "); display_rate(start, end, num_render * METATILE * METATILE); printf("Total tiles handled: "); display_rate(start, end, num_all); return 0; } #endif