src/http/modules/download/ngx_http_download_module.c

Go to the documentation of this file.

/*
 *  Copyright 2009 DreamLab Onet.pl Sp. z o.o.
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 *   Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *   limitations under the License.
 *
 *
 * Authors:
 * Przemyslaw Gajda, email: quermit(malpa)gmail.com
 * Piotr Janik, email: janik.piotrek(malpa)gmail.com
 *                                                                       malpa = @
 */

#include <ngx_config.h>
#include <ngx_core.h>
#include <ngx_http.h>
#include <ngx_log.h>


#include <libxml/parser.h>
#include <libxml/tree.h>

#include "ngx_http_download_module.h"


static void *
ngx_http_download_create_conf(ngx_conf_t *cf);
static char *
ngx_http_download_merge_conf(ngx_conf_t *cf, void *parent, void *child);

static ngx_command_t  ngx_http_download_commands[] = {
                {ngx_string( "download_server_timeout" ),
                NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
                ngx_conf_set_msec_slot,
                NGX_HTTP_LOC_CONF_OFFSET,
                offsetof(ngx_http_download_conf_t, server_timeout),
                NULL},

                ngx_null_command
};

static ngx_http_module_t  ngx_http_download_module_ctx = {
                NULL,NULL,NULL,NULL,NULL,NULL,
                ngx_http_download_create_conf,
                ngx_http_download_merge_conf
};

ngx_module_t  ngx_http_download_module = {
                NGX_MODULE_V1,
                &ngx_http_download_module_ctx, /* module context */
                ngx_http_download_commands,   /* module directives */
                NGX_HTTP_MODULE,                           /* module type */
                NULL,                                             /* init master */
                NULL,                                             /* init module */
                NULL,                                             /* init process */
                NULL,                                             /* init thread */
                NULL,                                             /* exit thread */
                NULL,                                             /* exit process */
                NULL,                                             /* exit master */
                NGX_MODULE_V1_PADDING
};

// ==================================================================================


static ngx_int_t
ngx_http_download_create_request(ngx_http_download_conn_t *conn);
static ngx_int_t
ngx_http_download_flush_buffer(ngx_http_download_conn_t *conn, ssize_t bytes);
static void
ngx_http_download_write_handler(ngx_event_t *wev);
static void
ngx_http_download_read_handler(ngx_event_t *rev);
static void
ngx_http_download_dummy_handler(ngx_event_t *ev);
static void
ngx_http_download_parse_headers(ngx_http_download_conn_t *conn);
static void
ngx_http_download_finalize_connection(ngx_http_download_conn_t *conn, ngx_event_t *e, ngx_int_t rc);

ngx_http_download_conn_t *
ngx_http_download_create_new(ngx_http_request_t *r)
{
        ngx_http_download_conn_t        *conn;
        ngx_http_download_conf_t        *conf;
        ngx_pool_t                                      *pool;
        ngx_log_t                                       *dl_log;
        ngx_log_t                                       *log;

        conf = ngx_http_get_module_loc_conf(r, ngx_http_download_module);

        log = r->connection->log;

        pool = ngx_create_pool(BUFFER_SIZE, log);
        if( pool == NULL ) {
                return NULL;
        }

        dl_log = ngx_palloc(pool, sizeof(ngx_log_t));
        if(dl_log == NULL) {
                return NULL;
        }
        ngx_memcpy(dl_log, log, sizeof(ngx_log_t));
        pool->log = dl_log;

        conn = ngx_pcalloc(pool, sizeof(ngx_http_download_conn_t));
        if( conn == NULL ) {
                return NULL;
        }

        // conn initialization
        conn->r = r;
        conn->pool = pool;
        conn->rpool = r->pool; // pool
        conn->connect_timeout = conf->server_timeout;
        conn->read_timeout = conf->server_timeout;
        conn->write_timeout = conf->server_timeout;

        conn->log = dl_log;

        ngx_log_debug1(NGX_LOG_DEBUG_HTTP, conn->log, 0,
                        "[download_create_new] | def timeout: %d", conf->server_timeout);

        return conn;
}

ngx_int_t 
ngx_http_download_enqueue(ngx_http_download_conn_t *conn)
{
        ngx_int_t                                       rc;
        ngx_http_request_t                      *r;
        ngx_connection_t                        *c;
        ngx_peer_connection_t           *peer;
        struct  hostent                         *hp;
        struct sockaddr_in                      *sin;

        r = conn->r;
        ngx_log_debug3(NGX_LOG_DEBUG_HTTP, conn->log, 0,
                        "[download_enqueue] host: %s, port: %d, timeout: %d", conn->host, conn->port, conn->connect_timeout);
        
        conn->done = 0;
        conn->response_length = 0;
        conn->out_bufs = NULL;
        conn->out_bufs_tail = NULL;

        peer = ngx_pcalloc(conn->pool, sizeof(ngx_peer_connection_t));
        if( peer == NULL ) {
                return NGX_ERROR;
        }

        conn->buffer = ngx_create_temp_buf(conn->rpool, BUFFER_SIZE);

        if( conn->buffer == NULL ) {
                return NGX_ERROR;
        }

        peer->sockaddr =  ngx_pcalloc(conn->pool, sizeof(struct sockaddr_in));
        if( peer->sockaddr == NULL ) {
                return NGX_ERROR;
        }

        if( conn->request == NULL ) {
                if( ngx_http_download_create_request(conn) == NGX_ERROR ) {
                        return NGX_ERROR;
                }
        }
        // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! PROXY !!!!!!!!!!!!
        //if( ngx_strncmp(conn->host, "127.0.0.1", 9) != 0 ) {
        //      conn->host = (u_char*)"10.166.21.2";
        //      conn->port = 8080;
        //}
        // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

        sin = (struct sockaddr_in *)peer->sockaddr;
        hp = gethostbyname((const char*)conn->host);
        if( hp == NULL || hp->h_addr_list[0] == NULL ) {
                return NGX_ERROR;
        }
        bcopy(hp->h_addr, &(sin->sin_addr.s_addr), hp->h_length);
        sin->sin_family = AF_INET;
        sin->sin_port = htons(conn->port);

        peer->socklen = sizeof(struct sockaddr_in);
        peer->name = ngx_pcalloc(conn->pool, sizeof(ngx_str_t));
        peer->name->data = (u_char*)"download_module";
        peer->name->len = sizeof(peer->name->data)-1;
        peer->get = ngx_event_get_peer;
        peer->log = conn->log;//r->connection->log;
        peer->log_error = NGX_ERROR_ERR;
        
        // establishing connection
        rc = ngx_event_connect_peer(peer);
        
        if( rc == NGX_ERROR || rc == NGX_BUSY || rc == NGX_DECLINED ) {
                ngx_http_download_finalize_connection(conn, NULL, rc);
                ngx_log_debug1(NGX_LOG_DEBUG_HTTP, conn->log, 0,
                                "[download_enqueue] can't connect: %d", rc);
                return rc;
        }
        else {
                conn->p = peer;
                ngx_log_debug1(NGX_LOG_DEBUG_HTTP, conn->log, 0,
                                "[download_enqueue] connect_peer rc: %d (OK)", rc);
        }

        c = peer->connection;
        c->read->handler = ngx_http_download_read_handler;
        c->write->handler = ngx_http_download_write_handler;
        c->read->data = conn;   
        c->write->data = conn;
        c->data = r;

        ngx_add_timer(c->read, conn->read_timeout);
        ngx_add_timer(c->write, conn->write_timeout);

        ngx_http_download_write_handler(c->write);

        return rc;
}

ngx_int_t
ngx_http_download_parse_url(ngx_str_t *url, ngx_http_download_conn_t *conn)
{
        u_char                  *host, *port, *last, *uri, *args;
        size_t                  len;
        ngx_int_t               n;

        if( url->len < 7 ) {
                conn->err = (u_char*)"bad url";
                return NGX_ERROR;
        }

        host = url->data;
        last = host + url->len;

        if( strncmp("http://", (char*)host, 7) == 0 ) {
                host += 7;
        }
        else {
                conn->err = (u_char*)"bad url";
                return NGX_ERROR;
        }

        port = ngx_strlchr(host, last, ':');
        uri = ngx_strlchr(host, last, '/');
        args = ngx_strlchr(host, last, '?');

        if (args) {
                if (uri == NULL) {
                        uri = args;
                } else if (args < uri) {
                        uri = args;
                }
        }

        if (uri) {
                conn->uri = ngx_pcalloc(conn->pool, last-uri+1);
                if( conn->uri == NULL ) {
                        conn->err = (u_char*)"ngx_calloc error";
                        return NGX_ERROR;
                }
                ngx_cpystrn(conn->uri, uri, last-uri+1);

                last = uri;

                if (uri < port) {
                        port = NULL;
                }
        }

        if (port) {
                port++;

                len = last - port;

                if (len == 0) {
                        conn->err = (u_char*)"invalid port";
                        return NGX_ERROR;
                }

                n = ngx_atoi(port, len);

                if (n < 1 || n > 65536) {
                        conn->err = (u_char*)"invalid port";
                        return NGX_ERROR;
                }

                conn->port = n;

                last = port - 1;
        }                

        len = last - host;

        if (len == 0) {
                conn->err = (u_char*)"no host";
                return NGX_ERROR;
        }

        if (len == 1 && *host == '*') {
                len = 0;
        }

        conn->host = ngx_pcalloc(conn->pool, len+1);
        if( conn->host == NULL ) {
                conn->err = (u_char*)"ngx_calloc error";
                return NGX_ERROR;
        }
        ngx_cpystrn(conn->host, host, len+1);

        if (!port) {
                conn->port = 80; // default port
        }

        return NGX_OK;
}

void
ngx_http_download_cleanup(ngx_http_download_conn_t *conn)
{
        if( conn->done || conn->p==NULL ) { // connection finalized or error while establishing connection
                ngx_log_debug0(NGX_LOG_DEBUG_HTTP, conn->log, 0,
                                "[download_cleanup] destroying pool");

                if( conn->pool ) {
                        ngx_destroy_pool(conn->pool);
                }
        }
        else { // connection is active
                ngx_log_debug0(NGX_LOG_DEBUG_HTTP, conn->log, 0,
                                "[download_cleanup] terminating download");
                conn->terminated = 1;
                conn->done_handler = NULL;
        }
}

static ngx_int_t
ngx_http_download_create_request(ngx_http_download_conn_t *conn)
{
        ngx_buf_t       *req;
        size_t          size;

        ngx_log_debug0(NGX_LOG_DEBUG_HTTP, conn->log, 0,
                        "[download_create_request]");

        size = sizeof("GET ")-1 + strlen((char*)conn->uri) + sizeof(" HTTP/1.1\r\n")-1 + \
                        sizeof("Host: ")-1 + strlen((char*)conn->host) + sizeof("\r\n\r\n")-1;

        req = conn->request = ngx_create_temp_buf(conn->pool, size + 1);
        if( req == NULL ) {
                return NGX_ERROR;
        }
        
        // HTTP 1.0 - no keep alive
        req->last = req->pos + size;
        ngx_sprintf((u_char*)req->pos, "GET %s HTTP/1.0\r\nHost: %s\r\n\r\n", conn->uri, conn->host);
        req->pos[size] = '\0';

        //ngx_log_debug2(NGX_LOG_DEBUG_HTTP, conn->log, 0,
        //              "[download_create_request] done (%d): \n%s", size, (u_char*)req->pos);

        return NGX_OK;
}

static ngx_int_t
ngx_http_download_flush_buffer(ngx_http_download_conn_t *conn, ssize_t bytes)
{
        //ngx_buf_t                             *b;
        ngx_chain_t                             *cl;
        ngx_http_request_t              *r;


        cl = NULL;
        r = conn->r;
        if( conn->out_bufs == NULL ) { // first buffer
                cl = conn->out_bufs = conn->out_bufs_tail = ngx_alloc_chain_link(conn->rpool);
                if( cl == NULL ) {
                        return NGX_ERROR;
                }               
        }

        if (cl == NULL) { // there was a buffer
                cl = conn->out_bufs_tail->next = ngx_alloc_chain_link(conn->rpool);
                if( cl == NULL ) {
                        return NGX_ERROR;
                }
                conn->out_bufs_tail = cl;
        }

        cl->next = NULL;
        cl->buf = conn->buffer;
        conn->buffer = NULL;

        cl->buf->flush = 1;
        cl->buf->memory = 1;

        return NGX_OK;
}

static void
ngx_http_download_write_handler(ngx_event_t *wev)
{
        ngx_connection_t                        *c;
        ssize_t                                         n, size;
        ngx_http_request_t                      *r;
        ngx_http_download_conn_t        *conn;
        ngx_peer_connection_t           *p;
        
        conn = wev->data;
        p = conn->p;
        c = p->connection;
        r = conn->r;

        //ngx_log_debug0(NGX_LOG_DEBUG_HTTP, conn->log, 0,
        //              "ngx_http_download_write_handler");

        if( conn->terminated ) {
                ngx_log_debug0(NGX_LOG_DEBUG_HTTP, conn->log, 0,
                                "[download_write_handler] terminated");

                ngx_http_download_finalize_connection(conn, wev, NGX_ERROR);
                return ;
        }

        if( wev->timedout ) {
                ngx_log_error(NGX_LOG_ERR, conn->log, 0,
                                "[download_write_handler] timeout :: %s", conn->host);
                ngx_http_download_finalize_connection(conn, wev, NGX_ETIMEDOUT);
                return ;        
        }

        size = conn->request->last - conn->request->pos;

        //ngx_log_debug1(NGX_LOG_DEBUG_HTTP, conn->log, 0,
        //              "ngx_http_download_write_handler - sending: %d", size);

        n = ngx_send(c, (u_char*)conn->request->pos, size );

        //ngx_log_debug1(NGX_LOG_DEBUG_HTTP, conn->log, 0,
        //              "ngx_http_download_write_handler - sent: %d", n);

        if (n == NGX_ERROR) {
                ngx_http_download_finalize_connection(conn, wev, NGX_ERROR);
                return;
        }

        if (n > 0) {
                conn->request->pos += n;
                if( n == size ) { // everything has been written
                        c->write->handler = ngx_http_download_dummy_handler;
                        
                        if (wev->timer_set) {
                                ngx_del_timer(wev);
                        }

                        if( ngx_handle_write_event(wev, 0) == NGX_ERROR ) {
                                ngx_http_download_finalize_connection(conn, wev, NGX_ERROR);
                        }

                        return ;
                }
        }
        
        if( !wev->timer_set ) {
                ngx_add_timer(wev, conn->write_timeout);
        }
}

static void ngx_http_download_dummy_handler(ngx_event_t *ev)
{
}

static void
ngx_http_download_read_handler(ngx_event_t *rev)
{
        size_t                                          size;
        ngx_int_t                                       n;
        ngx_buf_t                                       *b;
        ngx_connection_t                        *c;
        ngx_http_request_t                      *r;
        ngx_peer_connection_t           *p;
        ngx_http_download_conn_t        *conn;
        
        conn = rev->data;
        p = conn->p;
        c = p->connection;
        r = conn->r;
        

        if( conn->terminated ) {
                ngx_log_debug0(NGX_LOG_DEBUG_HTTP, conn->log, 0,
                                "[download_read_handler] terminated");

                ngx_http_download_finalize_connection(conn, rev, NGX_ERROR);
                return ;
        }

        if (rev->timedout) {
                ngx_connection_error(c, NGX_ETIMEDOUT, "download timed out");
                ngx_log_error(NGX_LOG_ERR, conn->log, 0,
                                "[download_read_handler] timeout :: %s", conn->host);
                ngx_http_download_finalize_connection(conn, rev, NGX_ETIMEDOUT);
                return;
        }

        //b = &u->buffer;

        for ( ;; ) {

                b = conn->buffer;
                size = b->end - b->last;
                if( size == 0 ) {
                        if( ngx_http_download_flush_buffer(conn, b->end - b->start) == NGX_ERROR ) {
                                ngx_http_download_finalize_connection(conn, rev, NGX_ERROR);
                                return;
                        }

                        conn->buffer = b = ngx_create_temp_buf(conn->rpool, BUFFER_SIZE);

                        if(b == NULL)    
                        {
                                ngx_log_error(NGX_LOG_ERR, conn->log, 0,
                                                "[download_read_handler] create temp buf failed");

                                ngx_http_download_finalize_connection(conn, rev, NGX_ERROR);
                                return;
                        }
                        size = b->end - b->last;
                }


                n = c->recv(c, b->last, size);

                //ngx_log_debug1(NGX_LOG_DEBUG_HTTP, conn->log, 0,
                //              "ngx_http_download_read_handler: recv %d", n);

                if (n == NGX_AGAIN) {
                        break;
                }

                if (n == 0 || n == NGX_ERROR) {
                        size = b->last - b->pos;
                        if( ngx_http_download_flush_buffer(conn, size) == NGX_ERROR ) {
                                ngx_http_download_finalize_connection(conn, rev, NGX_ERROR);
                                return;
                        }
                        ngx_http_download_finalize_connection(conn, rev, n);
                        return;
                }

                if( n>0 ) {
                        conn->response_length += n;
                        b->last += n;
                }

                if (!rev->ready) {
                        break;
                }
        }

        if (ngx_handle_read_event(rev, 0) != NGX_OK) {
                ngx_http_download_finalize_connection(conn, rev, NGX_ERROR);
                return;
        }

        if (rev->active) {
                ngx_add_timer(rev, conn->read_timeout);
        } else if (rev->timer_set) {
                ngx_del_timer(rev);
        }
}

static void
ngx_http_download_parse_headers(ngx_http_download_conn_t *conn)
{
        u_char          *s;
        u_char          *end;

        u_char          http[9];
        ngx_int_t       status;

        s = conn->out_bufs->buf->pos;
        end = conn->body_start;

        conn->status = -1;

        if( end - s < 12 ) {
                return ;
        }

        if( sscanf((char*)s, "%8s %d", http, (int *) &status) != 2 ) {
                return ;
        }

        if( ngx_strlen(http) != 8 || (ngx_strncmp(http, "HTTP/1.1", 8)!=0 && ngx_strncmp(http, "HTTP/1.0", 8)!=0) ) {
                return ;
        }

        conn->status = status;
}

static void
ngx_http_download_finalize_connection(ngx_http_download_conn_t *conn, ngx_event_t *e, ngx_int_t rc)
{
        ngx_peer_connection_t   *p;
        ngx_http_request_t              *r;

        p = conn->p;
        r = conn->r;

        ngx_log_debug1(NGX_LOG_DEBUG_HTTP, conn->log, 0,
                                "[download_finalize_connection] rc: %d", rc);

        if ( !conn->done ) {

                if( p->connection ) {
                        ngx_log_debug1(NGX_LOG_DEBUG_HTTP, conn->log, 0,
                                        "[download_finalize_connection] close download connection: %d", p->connection->fd);

                        if (p->connection->read->timer_set) {
                                ngx_del_timer(p->connection->read);
                        }
                        if (p->connection->write->timer_set) {
                                ngx_del_timer(p->connection->write);
                        }

                        ngx_close_connection(p->connection);

                        p->connection = NULL;
                }

                conn->done = 1;
                conn->error = rc;
        
                if( conn->out_bufs && conn->out_bufs->buf ) {
                        conn->body_start = (u_char*)strstr((char*)conn->out_bufs->buf->pos, "\r\n\r\n");
                        if( conn->body_start ) {
                                conn->body_start += 4;
                        }

                        if( conn->body_start ) {
                                ngx_http_download_parse_headers(conn);
                        }

                        if( conn->out_bufs_tail && conn->out_bufs_tail->buf ) {
                                conn->out_bufs_tail->buf->last_buf = 1;
                                conn->out_bufs_tail->buf->last_in_chain = 1;
                        }
                }

                //callback
                if( conn->done_handler != NULL ) {
                        conn->done_handler(conn);
                }
                else {
                        ngx_http_download_cleanup(conn);
                }
        }
}

static void *
ngx_http_download_create_conf(ngx_conf_t *cf)
{
        ngx_http_download_conf_t        *conf;

        ngx_log_debug0(NGX_LOG_DEBUG_HTTP, cf->log, 0,
                        "ngx_http_download_create_conf");

        conf = ngx_pcalloc(cf->pool, sizeof(ngx_http_download_conf_t));
        if (conf == NULL) {
                return NGX_CONF_ERROR;
        }

        conf->server_timeout = NGX_CONF_UNSET_MSEC;

        conf->log = ngx_palloc(cf->pool, sizeof(ngx_log_t));
        ngx_memcpy(conf->log, cf->log, sizeof(ngx_log_t));

        return conf;
}


static char *
ngx_http_download_merge_conf(ngx_conf_t *cf, void *parent, void *child)
{
        ngx_http_download_conf_t                *prev = parent;
        ngx_http_download_conf_t                *conf = child;

        ngx_conf_merge_msec_value(conf->server_timeout, prev->server_timeout, 1000);

        if (conf->server_timeout <= 0) {
           ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
                   "check_server_timeout must be greater then zero");
           return NGX_CONF_ERROR;
        }

        return NGX_CONF_OK;
}

---