ref: 3d78feffcbbc4b2977faf2526d9cc40f8aaca16a
parent: 009ecafbe707ad6ad0f1a8e55b99f6e8cd4bf341
author: Timothy B. Terriberry <[email protected]>
date: Fri Oct 12 11:29:52 EDT 2012
Major updates to the http backend. * Now supports HTTP/1.1 persistent connections with pipelining. This speeds up chain enumeration on large files by almost a factor of 2 over http, and by roughly a factor of 4 over https. The difference between http and https is now much smaller. * Add timeouts to all the socket I/O. * Estimate the number of available bytes to read and use it when making connection re-use decisions. * Add support for https with proxies using HTTP/1.1 CONNECT tunnels. * Fix TLS session re-use (it requires clean shutdown). * Various other code re-organization and minor improvements.
--- a/include/opusfile.h
+++ b/include/opusfile.h
@@ -673,6 +673,8 @@
size_t _size,int *_error);
/**Open a stream from a URL.
+ See the security warning in op_open_url_with_proxy() for information about
+ possible truncation attacks with HTTPS.
\param _url The URL to open.
Currently only the <file:>, <http:>, and <https:> schemes
are supported.
@@ -688,6 +690,18 @@
int _flags,int *_error) OP_ARG_NONNULL(1);
/**Open a stream from a URL using the specified proxy.
+ \warning HTTPS streams that are not served with a Content-Length header may
+ be vulnerable to truncation attacks.
+ The abstract stream interface is incapable of signaling whether a connection
+ was closed gracefully (with a TLS "close notify" message) or abruptly (and,
+ e.g., possibly by an attacker).
+ If you wish to guarantee that you are not vulnerable to such attacks, you
+ might consider only allowing seekable streams (which must have a valid
+ content length) and verifying the file position reported by op_raw_tell()
+ after decoding to the end is at least as large as that reported by
+ op_raw_total() (though possibly larger).
+ However, this approach will not work for live streams or HTTP/1.0 servers
+ (which do not support Range requets).
\param _url The URL to open.
Currently only the <file:>, <http:>, and <https:>
schemes are supported.
--- a/src/http.c
+++ b/src/http.c
@@ -15,6 +15,24 @@
#include <limits.h>
#include <string.h>
+/*RFCs referenced in this file:
+ RFC 1738: Uniform Resource Locators (URL)
+ RFC 1945: Hypertext Transfer Protocol -- HTTP/1.0
+ RFC 2068: Hypertext Transfer Protocol -- HTTP/1.1
+ RFC 2145: Use and Interpretation of HTTP Version Numbers
+ RFC 2246: The TLS Protocol Version 1.0
+ RFC 2616: Hypertext Transfer Protocol -- HTTP/1.1
+ RFC 2617: HTTP Authentication: Basic and Digest Access Authentication
+ RFC 2817: Upgrading to TLS Within HTTP/1.1
+ RFC 2818: HTTP Over TLS
+ RFC 3492: Punycode: A Bootstring encoding of Unicode for Internationalized
+ Domain Names in Applications (IDNA)
+ RFC 3986: Uniform Resource Identifier (URI): Generic Syntax
+ RFC 3987: Internationalized Resource Identifiers (IRIs)
+ RFC 5894: Internationalized Domain Names for Applications (IDNA):
+ Background, Explanation, and Rationale
+ RFC 6066: Transport Layer Security (TLS) Extensions: Extension Definitions*/
+
typedef struct OpusParsedURL OpusParsedURL;
typedef struct OpusStringBuf OpusStringBuf;
typedef struct OpusHTTPConn OpusHTTPConn;
@@ -48,14 +66,6 @@
return _s;
}
-/*Is this an https URL?
- For now we can simply check the last letter.*/
-#define OP_URL_IS_SSL(_url) ((_url)->scheme[4]=='s')
-
-#define OP_URL_IS_DEFAULT_PORT(_url) \
- (!OP_URL_IS_SSL(_url)&&(_url)->port==80 \
- ||OP_URL_IS_SSL(_url)&&(_url)->port==443)
-
/*URI character classes (from RFC 3986).*/
#define OP_URL_ALPHA \
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
@@ -184,11 +194,13 @@
}
#if defined(OP_ENABLE_HTTP)
+# include <sys/ioctl.h>
# include <sys/types.h>
# include <sys/socket.h>
# include <sys/timeb.h>
# include <arpa/inet.h>
# include <netinet/in.h>
+# include <netinet/tcp.h>
# include <fcntl.h>
# include <netdb.h>
# include <poll.h>
@@ -195,6 +207,77 @@
# include <unistd.h>
# include <openssl/ssl.h>
+/*The maximum number of simultaneous connections.
+ RFC 2616 says this SHOULD NOT be more than 2, but everyone on the modern web
+ ignores that (e.g., IE 8 bumped theirs up from 2 to 6, Firefox uses 15).
+ If it makes you feel better, we'll only ever actively read from one of these
+ at a time.
+ The others are kept around mainly to avoid slow-starting a new connection
+ when seeking, and time out rapidly.*/
+# define OP_NCONNS_MAX (4)
+
+/*The number of redirections at which we give up.
+ The value here is the current default in Firefox.
+ RFC 2068 mandated a maximum of 5, but RFC 2616 relaxed that to "a client
+ SHOULD detect infinite redirection loops."
+ Fortunately, 20 is less than infinity.*/
+# define OP_REDIRECT_LIMIT (20)
+
+/*The maximum size of a response message (before the body).
+ Responses larger than this will be discarded.
+ The buffer for this is currently stack-allocated, which will have to change
+ if you want to make it much larger.*/
+# define OP_RESPONSE_SIZE_MAX (1024)
+
+/*The number of milliseconds we will allow a connection to sit idle before we
+ refuse to resurrect it.
+ Apache as of 2.2 has reduced its default timeout to 5 seconds (from 15), so
+ that's what we'll use here.*/
+# define OP_CONNECTION_IDLE_TIMEOUT_MS (5*1000)
+
+/*The number of milliseconds we will wait to send or receive data before giving
+ up.*/
+# define OP_POLL_TIMEOUT_MS (30*1000)
+
+/*We will always attempt to read ahead at least this much in preference to
+ opening a new connection.*/
+# define OP_READAHEAD_THRESH_MIN (32*(opus_int32)1024)
+/*The amount to read ahead per iteration of the read-ahead loop.
+ 16 kB is the largest size OpenSSL will return at once.*/
+# define OP_READAHEAD_CHUNK_SIZE (16*1024)
+
+/*The amount of data to request after a seek.
+ This is a trade-off between read throughput after a seek vs. the the ability
+ to quickly perform another seek with the same connection.*/
+# define OP_PIPELINE_CHUNK_SIZE (32*(opus_int32)1024)
+/*Subsequent chunks are requested with larger and larger sizes until they pass
+ this threshold, after which we just ask for the rest of the resource.*/
+# define OP_PIPELINE_CHUNK_SIZE_MAX (1024*(opus_int32)1024)
+/*This is the maximum number of requests we'll make with a single connection.
+ Many servers will simply disconnect after we attempt some number of requests,
+ possibly without sending a Connection: close header, meaning we won't
+ discover it until we try to read beyond the end of the current chunk.
+ We can reconnect when that happens, but this is slow.
+ Instead, we impose a limit ourselves (set to the default for Apache
+ installations and thus likely the most common value in use).*/
+# define OP_PIPELINE_MAX_REQUESTS (100)
+/*This should be the number of requests, starting from a chunk size of
+ OP_PIPELINE_CHUNK_SIZE and doubling each time, until we exceed
+ OP_PIPELINE_CHUNK_SIZE_MAX and just request the rest of the file.
+ We won't reuse a connection when seeking unless it has at least this many
+ requests left, to reduce the chances we'll have to open a new connection
+ while reading forward afterwards.*/
+# define OP_PIPELINE_MIN_REQUESTS (7)
+
+/*Is this an https URL?
+ For now we can simply check the last letter of the scheme.*/
+# define OP_URL_IS_SSL(_url) ((_url)->scheme[4]=='s')
+
+/*Does this URL use the default port for its scheme?*/
+# define OP_URL_IS_DEFAULT_PORT(_url) \
+ (!OP_URL_IS_SSL(_url)&&(_url)->port==80 \
+ ||OP_URL_IS_SSL(_url)&&(_url)->port==443)
+
struct OpusParsedURL{/*Either "http" or "https".*/
char *scheme;
@@ -216,7 +299,10 @@
/*Parse a URL.
This code is not meant to be fast: strspn() with large sets is likely to be
slow, but it is very convenient.
- It is meant to be RFC 3986-compliant.*/
+ It is meant to be RFC 3986-compliant.
+ We currently do not support IRIs (Internationalized Resource Identifiers,
+ RFC 3987).
+ Callers should translate them to URIs first.*/
static int op_parse_url_impl(OpusParsedURL *_dst,const char *_src){const char *scheme_end;
const char *authority;
@@ -424,6 +510,13 @@
return op_sb_append(_sb,_s,strlen(_s));
}
+static int op_sb_append_port(OpusStringBuf *_sb,unsigned _port){+ char port_buf[7];
+ OP_ASSERT(_port<=65535U);
+ sprintf(port_buf,":%u",_port);
+ return op_sb_append_string(_sb,port_buf);
+}
+
static int op_sb_append_nonnegative_int64(OpusStringBuf *_sb,opus_int64 _i){char digit;
int nbuf_start;
@@ -455,9 +548,60 @@
return ret;
}
+static struct addrinfo *op_resolve(const char *_host,unsigned _port){+ struct addrinfo *addrs;
+ struct addrinfo hints;
+ char service[6];
+ memset(&hints,0,sizeof(hints));
+ hints.ai_socktype=SOCK_STREAM;
+ OP_ASSERT(_port<=65535U);
+ sprintf(service,"%u",_port);
+ if(OP_LIKELY(!getaddrinfo(_host,service,&hints,&addrs)))return addrs;
+ return NULL;
+}
+
+static int op_sock_set_nonblocking(int _fd,int _nonblocking){+ int flags;
+ flags=fcntl(_fd,F_GETFL);
+ if(OP_UNLIKELY(flags<0))return flags;
+ if(_nonblocking)flags|=O_NONBLOCK;
+ else flags&=~O_NONBLOCK;
+ return fcntl(_fd,F_SETFL,flags);
+}
+
+/*Disable/enable write coalescing if we can.
+ We always send whole requests at once and always parse the response headers
+ before sending another one, so normally write coalescing just causes added
+ delay.*/
+static void op_sock_set_tcp_nodelay(int _fd,int _nodelay){+# if defined(TCP_NODELAY)&&(defined(IPPROTO_TCP)||defined(SOL_TCP))
+# if defined(IPPROTO_TCP)
+# define OP_SO_LEVEL IPPROTO_TCP
+# else
+# define OP_SO_LEVEL SOL_TCP
+# endif
+ int ret;
+ ret=setsockopt(_fd,OP_SO_LEVEL,TCP_NODELAY,&_nodelay,sizeof(_nodelay));
+ /*It doesn't really matter if this call fails, but it would be interesting
+ to hit a case where it does.*/
+ OP_ASSERT(!ret);
+# endif
+}
+
+/*A single physical connection to an HTTP server.
+ We may have several of these open at once.*/
struct OpusHTTPConn{/*The current position indicator for this connection.*/
opus_int64 pos;
+ /*The position where the current request will end, or -1 if we're reading
+ until EOF (an unseekable stream or the initial HTTP/1.0 request).*/
+ opus_int64 end_pos;
+ /*The position where next request we've sent will start, or -1 if we haven't
+ sent the next request yet.*/
+ opus_int64 next_pos;
+ /*The end of the next request or -1 if we requested the rest of the resource.
+ This is only set to a meaningful value if next_pos is not -1.*/
+ opus_int64 next_end;
/*The SSL connection, if this is https.*/
SSL *ssl_conn;
/*The next connection in either the LRU or free list.*/
@@ -470,9 +614,14 @@
opus_int64 read_rate;
/*The socket we're reading from.*/
int fd;
+ /*The number of remaining requests we are allowed on this connection.*/
+ int nrequests_left;
+ /*The chunk size to use for pipelining requests.*/
+ opus_int32 chunk_size;
};
static void op_http_conn_init(OpusHTTPConn *_conn){+ _conn->next_pos=-1;
_conn->ssl_conn=NULL;
_conn->next=NULL;
_conn->fd=-1;
@@ -481,17 +630,10 @@
static void op_http_conn_clear(OpusHTTPConn *_conn){if(_conn->ssl_conn!=NULL)SSL_free(_conn->ssl_conn);
/*SSL frees the BIO for us.*/
- if(_conn->fd!=-1)close(_conn->fd);
+ if(_conn->fd>=0)close(_conn->fd);
}
-/*The maximum number of simultaneous connections.
- RFC 2616 says this SHOULD NOT be more than 2, but everyone on the modern web
- ignores that.
- If it makes you feel better, we'll only ever be reading from one of these at
- a time; the others are kept around mainly to avoid slow-starting a new
- connection if we seek back near somewhere we were reading before.*/
-# define OP_NCONNS_MAX (4)
-
+/*The global stream state.*/
struct OpusHTTPStream{/*The list of connections.*/
OpusHTTPConn conns[OP_NCONNS_MAX];
@@ -499,17 +641,13 @@
SSL_CTX *ssl_ctx;
/*The cached session to reuse for future connections.*/
SSL_SESSION *ssl_session;
- /*The LRU list (ordered from MRU to LRU) of connections.*/
+ /*The LRU list (ordered from MRU to LRU) of currently connected
+ connections.*/
OpusHTTPConn *lru_head;
/*The free list.*/
OpusHTTPConn *free_head;
/*The URL to connect to.*/
OpusParsedURL url;
- /*The connection we're currently reading from.
- This can be -1 if no connection is active.*/
- int cur_conni;
- /*The estimated time required to open a new connection, in milliseconds.*/
- opus_int32 connect_rate;
/*Information about the address we connected to.*/
struct addrinfo addr_info;
/*The address we connected to.*/
@@ -520,16 +658,26 @@
} addr;
/*A buffer used to build HTTP requests.*/
OpusStringBuf request;
- /*The offset of the tail of the request.
- Only the offset in the Range: header appears after this.*/
- int request_tail;
- /*Whether or not the server supports range requests.*/
- int seekable;
+ /*A buffer used to build proxy CONNECT requests.*/
+ OpusStringBuf proxy_connect;
/*The Content-Length, if specified, or -1 otherwise.
- This will always be valid for seekable streams.*/
+ This will always be specified for seekable streams.*/
opus_int64 content_length;
/*The position indicator used when no connection is active.*/
opus_int64 pos;
+ /*The connection we're currently reading from.
+ This can be -1 if no connection is active.*/
+ int cur_conni;
+ /*Whether or not the server supports range requests.*/
+ int seekable;
+ /*Whether or not the server supports HTTP/1.1 with persistent connections.*/
+ int pipeline;
+ /*The offset of the tail of the request.
+ Only the offset in the Range: header appears after this, allowing us to
+ quickly edit the request to ask for a new range.*/
+ int request_tail;
+ /*The estimated time required to open a new connection, in milliseconds.*/
+ opus_int32 connect_rate;
};
static void op_http_stream_init(OpusHTTPStream *_stream){@@ -546,332 +694,54 @@
_stream->lru_head=NULL;
op_parsed_url_init(&_stream->url);
op_sb_init(&_stream->request);
+ op_sb_init(&_stream->proxy_connect);
_stream->seekable=0;
}
-/*Close the connection at the top of the LRU list.*/
-static void op_http_conn_close(OpusHTTPStream *_stream,OpusHTTPConn *_conn){+/*Close the connection and move it to the free list.
+ _stream: The stream containing the free list.
+ _conn: The connection to close.
+ _penxt: The linked-list pointer currently pointing to this connection.
+ _gracefully: Whether or not to shut down cleanly.*/
+static void op_http_conn_close(OpusHTTPStream *_stream,OpusHTTPConn *_conn,
+ OpusHTTPConn **_pnext,int _gracefully){+ /*If we don't shut down gracefully, the server MUST NOT re-use our session
+ according to RFC 2246, because it can't tell the difference between an
+ abrupt close and a truncation attack.
+ So we shut down gracefully if we can.
+ However, we will not wait if this would block (it's not worth the savings
+ from session resumption to do so).
+ Clients (that's us) MAY resume a TLS session that ended with an incomplete
+ close, according to RFC 2818, so that's no reason to make sure the server
+ shut things down gracefully.
+ It also says "client implementations MUST treat any premature closes as
+ errors and the data received as potentially truncated," but libopusfile
+ treats errors and potentially truncated data in unseekable streams just
+ like a normal EOF.
+ We warn about this in the docs, and give some suggestions if you truly want
+ to avoid truncation attacks.*/
+ if(_gracefully&&_conn->ssl_conn!=NULL)SSL_shutdown(_conn->ssl_conn);
op_http_conn_clear(_conn);
+ _conn->next_pos=-1;
_conn->ssl_conn=NULL;
_conn->fd=-1;
- OP_ASSERT(_stream->lru_head==_conn);
- _stream->lru_head=_conn->next;
+ OP_ASSERT(*_pnext==_conn);
+ *_pnext=_conn->next;
_conn->next=_stream->free_head;
_stream->free_head=_conn;
}
static void op_http_stream_clear(OpusHTTPStream *_stream){- while(_stream->lru_head!=NULL)op_http_conn_close(_stream,_stream->lru_head);
+ while(_stream->lru_head!=NULL){+ op_http_conn_close(_stream,_stream->lru_head,&_stream->lru_head,0);
+ }
if(_stream->ssl_session!=NULL)SSL_SESSION_free(_stream->ssl_session);
if(_stream->ssl_ctx!=NULL)SSL_CTX_free(_stream->ssl_ctx);
+ op_sb_clear(&_stream->proxy_connect);
op_sb_clear(&_stream->request);
op_parsed_url_clear(&_stream->url);
}
-static struct addrinfo *op_resolve(const char *_host,unsigned _port){- struct addrinfo *addrs;
- struct addrinfo hints;
- char service[6];
- memset(&hints,0,sizeof(hints));
- hints.ai_socktype=SOCK_STREAM;
- OP_ASSERT(_port<=65535U);
- sprintf(service,"%u",_port);
- if(OP_LIKELY(!getaddrinfo(_host,service,&hints,&addrs)))return addrs;
- return NULL;
-}
-
-static int op_sock_set_nonblocking(int _fd,int _nonblocking){- int flags;
- flags=fcntl(_fd,F_GETFL);
- if(OP_UNLIKELY(flags==-1))return flags;
- if(_nonblocking)flags|=O_NONBLOCK;
- else flags&=~O_NONBLOCK;
- return fcntl(_fd,F_SETFL,flags);
-}
-
-/*Try to start a connection to the next address in the given list of a given
- type.
- _fd: The socket to connect with.
- [inout] _addr: A pointer to the list of addresses.
- This will be advanced to the first one that matches the given
- address family (possibly the current one).
- _ai_family: The address family to connect to.
- Return: 1 If the connection was successful.
- 0 If the connection is in progress.
- OP_FALSE If the connection failed and there were no more addresses
- left to try.
- *_addr will be set to NULL in this case.*/
-static int op_sock_connect_next(int _fd,
- struct addrinfo **_addr,int _ai_family){- struct addrinfo *addr;
- addr=*_addr;
- for(;;){- /*Move to the next address of the requested type.*/
- for(;addr!=NULL&&addr->ai_family!=_ai_family;addr=addr->ai_next);
- *_addr=addr;
- /*No more: failure.*/
- if(addr==NULL)return OP_FALSE;
- if(connect(_fd,addr->ai_addr,addr->ai_addrlen)!=-1)return 1;
- if(OP_LIKELY(errno==EINPROGRESS))return 0;
- }
-}
-
-typedef int (*op_ssl_step_func)(SSL *_ssl_conn);
-
-/*Try to run an SSL function to completion (blocking if necessary).*/
-static int op_do_ssl_step(SSL *_ssl_conn,int _fd,op_ssl_step_func _step){- struct pollfd fd;
- fd.fd=_fd;
- for(;;){- int ret;
- int err;
- ret=(*_step)(_ssl_conn);
- if(ret>=0)return ret;
- err=SSL_get_error(_ssl_conn,ret);
- if(err==SSL_ERROR_WANT_READ)fd.events=POLLIN;
- else if(err==SSL_ERROR_WANT_WRITE)fd.events=POLLOUT;
- else return OP_FALSE;
- if(poll(&fd,1,-1)==-1)return OP_FALSE;
- }
-}
-
-/*The number of address families to try connecting to simultaneously.*/
-# define OP_NPROTOS (2)
-
-static int op_http_connect(OpusHTTPStream *_stream,OpusHTTPConn *_conn,
- struct addrinfo *_addrs,struct timeb *_start_time){- struct addrinfo *addr;
- struct addrinfo *addrs[OP_NPROTOS];
- struct pollfd fds[OP_NPROTOS];
- int ai_family;
- int nprotos;
- int ret;
- int pi;
- int pj;
- for(pi=0;pi<OP_NPROTOS;pi++)addrs[pi]=NULL;
- addr=_addrs;
- /*Try connecting via both IPv4 and IPv6 simultaneously, and keep the first
- one that succeeds.*/
- for(;addr!=NULL;addr=addr->ai_next){- /*Give IPv6 a slight edge by putting it first in the list.*/
- if(addr->ai_family==AF_INET6){- OP_ASSERT(addr->ai_addrlen<=sizeof(struct sockaddr_in6));
- if(addrs[0]==NULL)addrs[0]=addr;
- }
- else if(addr->ai_family==AF_INET){- OP_ASSERT(addr->ai_addrlen<=sizeof(struct sockaddr_in));
- if(addrs[1]==NULL)addrs[1]=addr;
- }
- }
- /*Consolidate the list of addresses.*/
- for(pi=nprotos=0;pi<OP_NPROTOS;pi++){- if(addrs[pi]!=NULL){- addrs[nprotos]=addrs[pi];
- nprotos++;
- }
- }
- ret=ftime(_start_time);
- OP_ASSERT(!ret);
- *&_conn->read_time=*_start_time;
- _conn->read_bytes=0;
- _conn->read_rate=0;
- /*Try to start a connection to each protocol.*/
- for(pi=0;pi<nprotos;pi++){- ai_family=addrs[pi]->ai_family;
- fds[pi].fd=socket(ai_family,SOCK_STREAM,addrs[pi]->ai_protocol);
- fds[pi].events=POLLOUT;
- if(OP_LIKELY(fds[pi].fd!=-1)){- if(OP_LIKELY(op_sock_set_nonblocking(fds[pi].fd,1)!=-1)){- ret=op_sock_connect_next(fds[pi].fd,addrs+pi,ai_family);
- if(ret>1){- /*It succeeded right away, so stop.*/
- nprotos=pi+1;
- break;
- }
- /*Otherwise go on to the next protocol, and skip the clean-up below.*/
- else if(ret==0)continue;
- /*Tried all the addresses for this protocol.*/
- }
- /*Clean up the socket.*/
- close(fds[pi].fd);
- }
- /*Remove this protocol from the list.*/
- memmove(addrs+pi,addrs+pi+1,sizeof(*addrs)*(nprotos-pi-1));
- nprotos--;
- pi--;
- }
- /*Wait for one of the connections to finish.*/
- while(pi>=nprotos&&nprotos>0&&poll(fds,nprotos,-1)!=-1){- for(pi=0;pi<nprotos;pi++){- socklen_t errlen;
- int err;
- /*Still waiting...*/
- if(!fds[pi].revents)continue;
- errlen=sizeof(err);
- if(getsockopt(fds[pi].fd,SOL_SOCKET,SO_ERROR,&err,&errlen)!=-1&&err==0){- /*Success!*/
- break;
- }
- /*Move on to the next address for this protocol.*/
- ai_family=addrs[pi]->ai_family;
- addrs[pi]=addrs[pi]->ai_next;
- ret=op_sock_connect_next(fds[pi].fd,addrs+pi,ai_family);
- /*It succeeded right away, so stop.*/
- if(ret>0)break;
- /*Otherwise go on to the next protocol, and skip the clean-up below.*/
- else if(ret==0)continue;
- /*Tried all the addresses for this protocol.
- Remove it from the list.*/
- close(fds[pi].fd);
- memmove(fds+pi,fds+pi+1,sizeof(*fds)*(nprotos-pi-1));
- memmove(addrs+pi,addrs+pi+1,sizeof(*addrs)*(nprotos-pi-1));
- nprotos--;
- pi--;
- }
- }
- /*Close all the other sockets.*/
- for(pj=0;pj<nprotos;pj++)if(pi!=pj)close(fds[pj].fd);
- /*If none of them succeeded, we're done.*/
- if(pi>=nprotos)return OP_FALSE;
- /*Save this address for future connection attempts.*/
- if(addrs[pi]!=&_stream->addr_info){- memcpy(&_stream->addr_info,addrs[pi],sizeof(_stream->addr_info));
- _stream->addr_info.ai_addr=&_stream->addr.s;
- _stream->addr_info.ai_next=NULL;
- memcpy(&_stream->addr,addrs[pi]->ai_addr,addrs[pi]->ai_addrlen);
- }
- if(OP_URL_IS_SSL(&_stream->url)){- SSL *ssl_conn;
- BIO *ssl_bio;
- /*Start the SSL connection.*/
- OP_ASSERT(_stream->ssl_ctx!=NULL);
- ssl_conn=SSL_new(_stream->ssl_ctx);
- if(OP_LIKELY(ssl_conn!=NULL)){- ssl_bio=BIO_new_socket(fds[pi].fd,BIO_NOCLOSE);
- if(OP_LIKELY(ssl_bio!=NULL)){-# if !defined(OPENSSL_NO_TLSEXT)
- /*Support for RFC 6066 Server Name Indication.*/
- SSL_set_tlsext_host_name(ssl_conn,_stream->url.host);
-# endif
- /*Resume a previous session if available.*/
- if(_stream->ssl_session!=NULL){- SSL_set_session(ssl_conn,_stream->ssl_session);
- }
- SSL_set_bio(ssl_conn,ssl_bio,ssl_bio);
- SSL_set_connect_state(ssl_conn);
- ret=op_do_ssl_step(ssl_conn,fds[pi].fd,SSL_connect);
- if(OP_LIKELY(ret>0)){- if(_stream->ssl_session==NULL){- /*Save a session for later resumption.*/
- ret=op_do_ssl_step(ssl_conn,fds[pi].fd,SSL_do_handshake);
- if(OP_LIKELY(ret>0)){- _stream->ssl_session=SSL_get1_session(ssl_conn);
- }
- }
- if(OP_LIKELY(ret>0)){- _conn->ssl_conn=ssl_conn;
- _conn->fd=fds[pi].fd;
- return 0;
- }
- }
- /*If we wanted to shut down cleanly, we would do:
- op_do_ssl_step(ssl_conn,fds[pi].fd,SSL_shutdown);*/
- }
- SSL_free(ssl_conn);
- }
- close(fds[pi].fd);
- return OP_FALSE;
- }
- /*Just a normal non-SSL connection.*/
- _conn->ssl_conn=NULL;
- _conn->fd=fds[pi].fd;
- return 0;
-}
-
-# define OP_BASE64_LENGTH(_len) (((_len)+2)/3*4)
-
-static const char BASE64_TABLE[64]={- 'A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P',
- 'Q','R','S','T','U','V','W','X','Y','Z','a','b','c','d','e','f',
- 'g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v',
- 'w','x','y','z','0','1','2','3','4','5','6','7','8','9','+','/'
-};
-
-static char *op_base64_encode(char *_dst,const char *_src,int _len){- unsigned s0;
- unsigned s1;
- unsigned s2;
- int ngroups;
- int i;
- ngroups=_len/3;
- for(i=0;i<ngroups;i++){- s0=_src[3*i+0];
- s1=_src[3*i+1];
- s2=_src[3*i+2];
- _dst[4*i+0]=BASE64_TABLE[s0>>2];
- _dst[4*i+1]=BASE64_TABLE[s0&3<<4|s1>>4];
- _dst[4*i+2]=BASE64_TABLE[s1&15<<2|s2>>6];
- _dst[4*i+3]=BASE64_TABLE[s2&63];
- }
- _len-=3*i;
- if(_len==1){- s0=_src[3*i+0];
- _dst[4*i+0]=BASE64_TABLE[s0>>2];
- _dst[4*i+1]=BASE64_TABLE[s0&3<<4];
- _dst[4*i+2]='=';
- _dst[4*i+3]='=';
- i++;
- }
- else if(_len==2){- s0=_src[3*i+0];
- s1=_src[3*i+1];
- _dst[4*i+0]=BASE64_TABLE[s0>>2];
- _dst[4*i+1]=BASE64_TABLE[s0&3<<4|s1>>4];
- _dst[4*i+2]=BASE64_TABLE[s1&15<<2];
- _dst[4*i+3]='=';
- i++;
- }
- _dst[4*i]='\0';
- return _dst+4*i;
-}
-
-/*Construct an HTTP authorization header using RFC 2617's Basic Authentication
- Scheme and append it to the given string buffer.*/
-static int op_sb_append_basic_auth_header(OpusStringBuf *_sb,
- const char *_header,const char *_user,const char *_pass){- int user_len;
- int pass_len;
- int user_pass_len;
- int base64_len;
- int nbuf_total;
- int ret;
- ret=op_sb_append_string(_sb,_header);
- ret|=op_sb_append(_sb,": Basic ",8);
- user_len=strlen(_user);
- pass_len=strlen(_pass);
- if(OP_UNLIKELY(pass_len>INT_MAX-user_len))return OP_EFAULT;
- if(OP_UNLIKELY(user_len+pass_len>(INT_MAX>>2)*3-3))return OP_EFAULT;
- user_pass_len=user_len+1+pass_len;
- base64_len=OP_BASE64_LENGTH(user_pass_len);
- /*Stick "user:pass" at the end of the buffer so we can Base64 encode it
- in-place.*/
- nbuf_total=_sb->nbuf;
- if(OP_UNLIKELY(base64_len>INT_MAX-nbuf_total))return OP_EFAULT;
- nbuf_total+=base64_len;
- ret|=op_sb_ensure_capacity(_sb,nbuf_total);
- if(OP_UNLIKELY(ret<0))return ret;
- _sb->nbuf=nbuf_total-user_pass_len;
- ret=op_sb_append(_sb,_user,user_len);
- OP_ASSERT(!ret);
- ret=op_sb_append(_sb,":",1);
- OP_ASSERT(!ret);
- ret=op_sb_append(_sb,_pass,pass_len);
- OP_ASSERT(!ret);
- op_base64_encode(_sb->buf+nbuf_total-base64_len,
- _sb->buf+nbuf_total-user_pass_len,user_pass_len);
- return op_sb_append(_sb,"\r\n",2);
-}
-
static int op_http_conn_write_fully(OpusHTTPConn *_conn,
const char *_buf,int _size){struct pollfd fd;
@@ -910,11 +780,29 @@
if(err!=EAGAIN&&err!=EWOULDBLOCK)return OP_FALSE;
fd.events=POLLOUT;
}
- if(poll(&fd,1,-1)==-1)return OP_FALSE;
+ if(poll(&fd,1,OP_POLL_TIMEOUT_MS)<=0)return OP_FALSE;
}
return 0;
}
+static int op_http_conn_estimate_available(OpusHTTPConn *_conn){+ int available;
+ int ret;
+ ret=ioctl(_conn->fd,FIONREAD,&available);
+ if(ret<0)available=0;
+ /*This requires the SSL read_ahead flag to be unset to work.
+ We ignore partial records as well as the protocol overhead for any pending
+ bytes.
+ This means we might return somewhat less than can truly be read without
+ blocking (if there's a partial record).
+ This is okay, because we're using this value to estimate network transfer
+ time, and we _have_ already received those bytes.
+ We also might return slightly more (due to protocol overhead), but that's
+ small enough that it probably doesn't matter.*/
+ if(_conn->ssl_conn!=NULL)available+=SSL_pending(_conn->ssl_conn);
+ return available;
+}
+
static opus_int32 op_time_diff_ms(const struct timeb *_end,
const struct timeb *_start){opus_int64 dtime;
@@ -926,7 +814,7 @@
return (opus_int32)dtime*1000+_end->millitm-_start->millitm;
}
-/*Update the read rate for this connection.*/
+/*Update the read rate estimate for this connection.*/
static void op_http_conn_read_rate_update(OpusHTTPConn *_conn){struct timeb read_time;
opus_int32 read_delta_ms;
@@ -949,9 +837,9 @@
/*Tries to read from the given connection.
[out] _buf: Returns the data read.
_size: The size of the buffer.
- _block Whether or not to block until some data is retrieved.*/
+ _blocking: Whether or not to block until some data is retrieved.*/
static ptrdiff_t op_http_conn_read(OpusHTTPConn *_conn,
- char *_buf,ptrdiff_t _size,int _block){+ char *_buf,ptrdiff_t _size,int _blocking){struct pollfd fd;
SSL *ssl_conn;
ptrdiff_t nread;
@@ -1001,9 +889,9 @@
_conn->read_bytes+=nread_unblocked;
op_http_conn_read_rate_update(_conn);
nread_unblocked=0;
- if(!_block)break;
+ if(!_blocking)break;
/*Need to wait to get any data at all.*/
- if(poll(&fd,1,-1)==-1)return 0;
+ if(poll(&fd,1,OP_POLL_TIMEOUT_MS)<=0)return 0;
}
while(nread<_size);
_conn->read_bytes+=nread_unblocked;
@@ -1011,7 +899,13 @@
}
/*Reads the entirety of a response to an HTTP request into a buffer.
- Actual parsing and validation is done later.*/
+ Actual parsing and validation is done later.
+ _buf: The buffer in which to read the response.
+ No terminating NUL is appended.
+ _size: The number of bytes available in the buffer.
+ Return: The number of bytes in the response on success, OP_EREAD if the
+ connection was closed before reading any data, or another negative
+ value on any other error.*/
static int op_http_conn_read_response(OpusHTTPConn *_conn,
char *_buf,int _size){/*The remaining size of the buffer.*/
@@ -1024,7 +918,7 @@
ptrdiff_t ret;
int len;
ret=op_http_conn_read(_conn,_buf,state,1);
- if(ret<=0)return OP_FALSE;
+ if(ret<=0)return _size==size?OP_EREAD:OP_FALSE;
/*We read some data.*/
_buf+=ret;
size-=ret;
@@ -1054,21 +948,10 @@
return OP_EIMPL;
}
-/*The number of redirections at which we give up.
- The value here is the current default in Firefox.
- RFC 2068 mandated a maximum of 5, but RFC 2616 relaxed that to "a client
- SHOULD detect infinite redirection loops."
- Fortunately, 20 is less than infinity.*/
-# define OP_REDIRECT_LIMIT (20)
-
-/*The maximum size of a response message (before the body).
- Responses larger than this will be discarded.*/
-# define OP_RESPONSE_SIZE_MAX (1024)
-
# define OP_HTTP_DIGIT "01234567890"
/*The Reason-Phrase is not allowed to contain control characters, except
- horizontal tab (HT).*/
+ horizontal tab (HT: \011).*/
# define OP_HTTP_CREASON_PHRASE \
"\001\002\003\004\005\006\007\010\012\013\014\015\016\017\020\021" \
"\022\023\024\025\026\027\030\031\032\033\034\035\036\037\177"
@@ -1077,8 +960,8 @@
"\001\002\003\004\005\006\007\010\011\012\013\014\015\016\017\020" \
"\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037\177"
-/*These also include ' ' and '\t', but we get those from CTLS.*/
-# define OP_HTTP_SEPARATORS "\"(),/:;<=>?@[\\]{}"+/*This also includes '\t', but we get that from OP_HTTP_CTLS.*/
+# define OP_HTTP_SEPARATORS " \"(),/:;<=>?@[\\]{}"/*TEXT can also include LWS, but that has structure, so we parse it
separately.*/
@@ -1094,9 +977,11 @@
}
}
-static char *op_http_parse_status_line(char **_status_code,char *_response){+static char *op_http_parse_status_line(int *_v1_1_compat,
+ char **_status_code,char *_response){char *next;
char *status_code;
+ int v1_1_compat;
size_t d;
/*RFC 2616 Section 6.1 does not say that the tokens in the Status-Line cannot
be separated by optional LWS, but since it specifically calls out where
@@ -1107,11 +992,26 @@
next=_response+4;
if(OP_UNLIKELY(*next++!='/'))return NULL;
d=strspn(next,OP_HTTP_DIGIT);
- if(OP_UNLIKELY(d<=0))return NULL;
- next+=d;
+ /*"Leading zeros MUST be ignored by recipients."*/
+ while(*next=='0'){+ next++;
+ OP_ASSERT(d>0);
+ d--;
+ }
+ /*We only support version 1.x*/
+ if(OP_UNLIKELY(d!=1)||OP_UNLIKELY(*next++!='1'))return NULL;
if(OP_UNLIKELY(*next++!='.'))return NULL;
d=strspn(next,OP_HTTP_DIGIT);
if(OP_UNLIKELY(d<=0))return NULL;
+ /*"Leading zeros MUST be ignored by recipients."*/
+ while(*next=='0'){+ next++;
+ OP_ASSERT(d>0);
+ d--;
+ }
+ /*We don't need to parse the version number.
+ Any non-zero digit means it's greater than 1.*/
+ v1_1_compat=d>0;
next+=d;
if(OP_UNLIKELY(*next++!=' '))return NULL;
status_code=next;
@@ -1123,10 +1023,25 @@
next+=strcspn(next,OP_HTTP_CREASON_PHRASE);
if(OP_UNLIKELY(*next++!='\r'))return NULL;
if(OP_UNLIKELY(*next++!='\n'))return NULL;
+ if(_v1_1_compat!=NULL)*_v1_1_compat=v1_1_compat;
*_status_code=status_code;
return next;
}
+/*Get the next response header.
+ [out] _header: The header token, NUL-terminated, with leading and trailing
+ whitespace stripped, and converted to lower case (to simplify
+ case-insensitive comparisons), or NULL if there are no more
+ response headers.
+ [out] _cdr: The remaining contents of the header, excluding the initial
+ colon (':') and the terminating CRLF ("\r\n"),+ NUL-terminated, and with leading and trailing whitespace
+ stripped, or NULL if there are no more response headers.
+ [inout] _s: On input, this points to the start of the current line of the
+ response headers.
+ On output, it points to the start of the first line following
+ this header, or NULL if there are no more response headers.
+ Return: 0 on success, or a negative value on failure.*/
static int op_http_get_next_header(char **_header,char **_cdr,char **_s){char *header;
char *header_end;
@@ -1203,7 +1118,7 @@
opus_int64 first;
opus_int64 last;
opus_int64 length;
- size_t d;
+ size_t d;
if(OP_UNLIKELY(op_strncasecmp(_cdr,"bytes",5)!=0))return OP_FALSE;
_cdr+=5;
d=op_http_lwsspn(_cdr);
@@ -1236,7 +1151,7 @@
}
if(OP_UNLIKELY(*_cdr!='\0'))return OP_FALSE;
if(OP_UNLIKELY(last<first))return OP_FALSE;
- if(length!=-1&&OP_UNLIKELY(last>=length))return OP_FALSE;
+ if(length>=0&&OP_UNLIKELY(last>=length))return OP_FALSE;
*_first=first;
*_last=last;
*_length=length;
@@ -1243,6 +1158,482 @@
return 0;
}
+/*Parse the Connection response header and look for a "close" token.
+ Return: 1 if a "close" token is found, 0 if it's not found, and a negative
+ value on error.*/
+static int op_http_parse_connection(char *_cdr){+ size_t d;
+ int ret;
+ ret=0;
+ for(;;){+ d=strcspn(_cdr,OP_HTTP_CTOKEN);
+ if(OP_UNLIKELY(d<=0))return OP_FALSE;
+ if(op_strncasecmp(_cdr,"close",(int)d)==0)ret=1;
+ /*We're supposed to strip and ignore any headers mentioned in the
+ Connection header if this response is from an HTTP/1.0 server (to
+ work around forwarding of hop-by-hop headers by old proxies), but the
+ only hop-by-hop header we look at is Connection itself.
+ Everything else is a well-defined end-to-end header, and going back and
+ undoing the things we did based on already-examined headers would be
+ hard (since we only scan them once, in a destructive manner).
+ Therefore we just ignore all the other tokens.*/
+ _cdr+=d;
+ d=op_http_lwsspn(_cdr);
+ if(d<=0)break;
+ _cdr+=d;
+ }
+ return OP_UNLIKELY(*_cdr!='\0')?OP_FALSE:0;
+}
+
+typedef int (*op_ssl_step_func)(SSL *_ssl_conn);
+
+/*Try to run an SSL function to completion (blocking if necessary).*/
+static int op_do_ssl_step(SSL *_ssl_conn,int _fd,op_ssl_step_func _step){+ struct pollfd fd;
+ fd.fd=_fd;
+ for(;;){+ int ret;
+ int err;
+ ret=(*_step)(_ssl_conn);
+ if(ret>=0)return ret;
+ err=SSL_get_error(_ssl_conn,ret);
+ if(err==SSL_ERROR_WANT_READ)fd.events=POLLIN;
+ else if(err==SSL_ERROR_WANT_WRITE)fd.events=POLLOUT;
+ else return OP_FALSE;
+ if(poll(&fd,1,OP_POLL_TIMEOUT_MS)<=0)return OP_FALSE;
+ }
+}
+
+/*Implement a BIO type that just indicates every operation should be retried.
+ We use this when initializing an SSL connection via a proxy to allow the
+ initial handshake to proceed all the way up to the first read attempt, and
+ then return.
+ This allows the TLS client hello message to be pipelined with the HTTP
+ CONNECT request.*/
+
+static int op_bio_retry_write(BIO *_b,const char *_buf,int _num){+ (void)_buf;
+ (void)_num;
+ BIO_clear_retry_flags(_b);
+ BIO_set_retry_write(_b);
+ return -1;
+}
+
+static int op_bio_retry_read(BIO *_b,char *_buf,int _num){+ (void)_buf;
+ (void)_num;
+ BIO_clear_retry_flags(_b);
+ BIO_set_retry_read(_b);
+ return -1;
+}
+
+static int op_bio_retry_puts(BIO *_b,const char *_str){+ return op_bio_retry_write(_b,_str,0);
+}
+
+static long op_bio_retry_ctrl(BIO *_b,int _cmd,long _num,void *_ptr){+ long ret;
+ (void)_b;
+ (void)_num;
+ (void)_ptr;
+ ret=0;
+ switch(_cmd){+ case BIO_CTRL_RESET:
+ case BIO_C_RESET_READ_REQUEST:{+ BIO_clear_retry_flags(_b);
+ /*Fall through.*/
+ }
+ case BIO_CTRL_EOF:
+ case BIO_CTRL_SET:
+ case BIO_CTRL_SET_CLOSE:
+ case BIO_CTRL_FLUSH:
+ case BIO_CTRL_DUP:{+ ret=1;
+ }break;
+ }
+ return ret;
+}
+
+static int op_bio_retry_new(BIO *_b){+ _b->init=1;
+ _b->num=0;
+ _b->ptr=NULL;
+ return 1;
+}
+
+static int op_bio_retry_free(BIO *_b){+ return _b!=NULL;
+}
+
+/*This is not const because OpenSSL doesn't allow it, even though it won't
+ write to it.*/
+static BIO_METHOD op_bio_retry_method={+ BIO_TYPE_NULL,
+ "retry",
+ op_bio_retry_write,
+ op_bio_retry_read,
+ op_bio_retry_puts,
+ NULL,
+ op_bio_retry_ctrl,
+ op_bio_retry_new,
+ op_bio_retry_free,
+ NULL
+};
+
+/*Establish a CONNECT tunnel and pipeline the start of the TLS handshake for
+ proxying https URL requests.*/
+int op_http_conn_establish_tunnel(OpusHTTPStream *_stream,
+ OpusHTTPConn *_conn,int _fd,SSL *_ssl_conn,BIO *_ssl_bio){+ char response[OP_RESPONSE_SIZE_MAX];
+ BIO *retry_bio;
+ char *status_code;
+ char *next;
+ int ret;
+ _conn->ssl_conn=NULL;
+ _conn->fd=_fd;
+ OP_ASSERT(_stream->proxy_connect.nbuf>0);
+ ret=op_http_conn_write_fully(_conn,
+ _stream->proxy_connect.buf,_stream->proxy_connect.nbuf);
+ if(OP_UNLIKELY(ret<0))return ret;
+ retry_bio=BIO_new(&op_bio_retry_method);
+ if(OP_UNLIKELY(retry_bio==NULL))return OP_EFAULT;
+ SSL_set_bio(_ssl_conn,retry_bio,_ssl_bio);
+ SSL_set_connect_state(_ssl_conn);
+ ret=SSL_connect(_ssl_conn);
+ /*This shouldn't succeed, since we can't read yet.*/
+ OP_ASSERT(ret<0);
+ SSL_set_bio(_ssl_conn,_ssl_bio,_ssl_bio);
+ /*Only now do we disable write coalescing, to allow the CONNECT
+ request and the start of the TLS handshake to be combined.*/
+ op_sock_set_tcp_nodelay(_fd,1);
+ ret=op_http_conn_read_response(_conn,response,sizeof(response));
+ if(OP_UNLIKELY(ret<0))return ret;
+ next=op_http_parse_status_line(NULL,&status_code,response);
+ /*According to RFC 2817, "Any successful (2xx) response to a
+ CONNECT request indicates that the proxy has established a
+ connection to the requested host and port.*/
+ if(OP_UNLIKELY(next==NULL)||OP_UNLIKELY(status_code[0]!='2'))return OP_FALSE;
+ return 0;
+}
+
+/*Perform the TLS handshake on a new connection.*/
+int op_http_conn_start_tls(OpusHTTPStream *_stream,OpusHTTPConn *_conn,
+ int _fd,SSL *_ssl_conn){+ BIO *ssl_bio;
+ int ret;
+ ssl_bio=BIO_new_socket(_fd,BIO_NOCLOSE);
+ if(OP_LIKELY(ssl_bio==NULL))return OP_FALSE;
+# if !defined(OPENSSL_NO_TLSEXT)
+ /*Support for RFC 6066 Server Name Indication.*/
+ SSL_set_tlsext_host_name(_ssl_conn,_stream->url.host);
+# endif
+ /*Resume a previous session if available.*/
+ if(_stream->ssl_session!=NULL){+ SSL_set_session(_ssl_conn,_stream->ssl_session);
+ }
+ /*If we're proxying, establish the CONNECT tunnel.*/
+ if(_stream->proxy_connect.nbuf>0){+ ret=op_http_conn_establish_tunnel(_stream,_conn,
+ _fd,_ssl_conn,ssl_bio);
+ if(OP_UNLIKELY(ret<0))return ret;
+ }
+ else{+ /*Otherwise, just use this socket directly.*/
+ op_sock_set_tcp_nodelay(_fd,1);
+ SSL_set_bio(_ssl_conn,ssl_bio,ssl_bio);
+ SSL_set_connect_state(_ssl_conn);
+ }
+ ret=op_do_ssl_step(_ssl_conn,_fd,SSL_connect);
+ if(OP_UNLIKELY(ret<=0))return OP_FALSE;
+ if(_stream->ssl_session==NULL){+ /*Save a session for later resumption.*/
+ ret=op_do_ssl_step(_ssl_conn,_fd,SSL_do_handshake);
+ if(OP_UNLIKELY(ret<=0))return OP_FALSE;
+ _stream->ssl_session=SSL_get1_session(_ssl_conn);
+ }
+ _conn->ssl_conn=_ssl_conn;
+ _conn->fd=_fd;
+ _conn->nrequests_left=OP_PIPELINE_MAX_REQUESTS;
+ return 0;
+}
+
+/*Try to start a connection to the next address in the given list of a given
+ type.
+ _fd: The socket to connect with.
+ [inout] _addr: A pointer to the list of addresses.
+ This will be advanced to the first one that matches the given
+ address family (possibly the current one).
+ _ai_family: The address family to connect to.
+ Return: 1 If the connection was successful.
+ 0 If the connection is in progress.
+ OP_FALSE If the connection failed and there were no more addresses
+ left to try.
+ *_addr will be set to NULL in this case.*/
+static int op_sock_connect_next(int _fd,
+ struct addrinfo **_addr,int _ai_family){+ struct addrinfo *addr;
+ addr=*_addr;
+ for(;;){+ /*Move to the next address of the requested type.*/
+ for(;addr!=NULL&&addr->ai_family!=_ai_family;addr=addr->ai_next);
+ *_addr=addr;
+ /*No more: failure.*/
+ if(addr==NULL)return OP_FALSE;
+ if(connect(_fd,addr->ai_addr,addr->ai_addrlen)>=0)return 1;
+ if(OP_LIKELY(errno==EINPROGRESS))return 0;
+ }
+}
+
+/*The number of address families to try connecting to simultaneously.*/
+# define OP_NPROTOS (2)
+
+static int op_http_connect(OpusHTTPStream *_stream,OpusHTTPConn *_conn,
+ struct addrinfo *_addrs,struct timeb *_start_time){+ struct addrinfo *addr;
+ struct addrinfo *addrs[OP_NPROTOS];
+ struct pollfd fds[OP_NPROTOS];
+ int ai_family;
+ int nprotos;
+ int ret;
+ int pi;
+ int pj;
+ for(pi=0;pi<OP_NPROTOS;pi++)addrs[pi]=NULL;
+ addr=_addrs;
+ /*Try connecting via both IPv4 and IPv6 simultaneously, and keep the first
+ one that succeeds.*/
+ for(;addr!=NULL;addr=addr->ai_next){+ /*Give IPv6 a slight edge by putting it first in the list.*/
+ if(addr->ai_family==AF_INET6){+ OP_ASSERT(addr->ai_addrlen<=sizeof(struct sockaddr_in6));
+ if(addrs[0]==NULL)addrs[0]=addr;
+ }
+ else if(addr->ai_family==AF_INET){+ OP_ASSERT(addr->ai_addrlen<=sizeof(struct sockaddr_in));
+ if(addrs[1]==NULL)addrs[1]=addr;
+ }
+ }
+ /*Consolidate the list of addresses.*/
+ for(pi=nprotos=0;pi<OP_NPROTOS;pi++){+ if(addrs[pi]!=NULL){+ addrs[nprotos]=addrs[pi];
+ nprotos++;
+ }
+ }
+ /*Pop the connection off the free list and put it on the LRU list.*/
+ OP_ASSERT(_stream->free_head==_conn);
+ _stream->free_head=_conn->next;
+ _conn->next=_stream->lru_head;
+ _stream->lru_head=_conn;
+ ret=ftime(_start_time);
+ OP_ASSERT(!ret);
+ *&_conn->read_time=*_start_time;
+ _conn->read_bytes=0;
+ _conn->read_rate=0;
+ /*Try to start a connection to each protocol.*/
+ for(pi=0;pi<nprotos;pi++){+ ai_family=addrs[pi]->ai_family;
+ fds[pi].fd=socket(ai_family,SOCK_STREAM,addrs[pi]->ai_protocol);
+ fds[pi].events=POLLOUT;
+ if(OP_LIKELY(fds[pi].fd>=0)){+ if(OP_LIKELY(op_sock_set_nonblocking(fds[pi].fd,1)>=0)){+ ret=op_sock_connect_next(fds[pi].fd,addrs+pi,ai_family);
+ if(ret>1){+ /*It succeeded right away, so stop.*/
+ nprotos=pi+1;
+ break;
+ }
+ /*Otherwise go on to the next protocol, and skip the clean-up below.*/
+ else if(ret==0)continue;
+ /*Tried all the addresses for this protocol.*/
+ }
+ /*Clean up the socket.*/
+ close(fds[pi].fd);
+ }
+ /*Remove this protocol from the list.*/
+ memmove(addrs+pi,addrs+pi+1,sizeof(*addrs)*(nprotos-pi-1));
+ nprotos--;
+ pi--;
+ }
+ /*Wait for one of the connections to finish.*/
+ while(pi>=nprotos&&nprotos>0&&poll(fds,nprotos,OP_POLL_TIMEOUT_MS)>0){+ for(pi=0;pi<nprotos;pi++){+ socklen_t errlen;
+ int err;
+ /*Still waiting...*/
+ if(!fds[pi].revents)continue;
+ errlen=sizeof(err);
+ if(getsockopt(fds[pi].fd,SOL_SOCKET,SO_ERROR,&err,&errlen)>=0&&err==0){+ /*Success!*/
+ break;
+ }
+ /*Move on to the next address for this protocol.*/
+ ai_family=addrs[pi]->ai_family;
+ addrs[pi]=addrs[pi]->ai_next;
+ ret=op_sock_connect_next(fds[pi].fd,addrs+pi,ai_family);
+ /*It succeeded right away, so stop.*/
+ if(ret>0)break;
+ /*Otherwise go on to the next protocol, and skip the clean-up below.*/
+ else if(ret==0)continue;
+ /*Tried all the addresses for this protocol.
+ Remove it from the list.*/
+ close(fds[pi].fd);
+ memmove(fds+pi,fds+pi+1,sizeof(*fds)*(nprotos-pi-1));
+ memmove(addrs+pi,addrs+pi+1,sizeof(*addrs)*(nprotos-pi-1));
+ nprotos--;
+ pi--;
+ }
+ }
+ /*Close all the other sockets.*/
+ for(pj=0;pj<nprotos;pj++)if(pi!=pj)close(fds[pj].fd);
+ /*If none of them succeeded, we're done.*/
+ if(pi>=nprotos)return OP_FALSE;
+ /*Save this address for future connection attempts.*/
+ if(addrs[pi]!=&_stream->addr_info){+ memcpy(&_stream->addr_info,addrs[pi],sizeof(_stream->addr_info));
+ _stream->addr_info.ai_addr=&_stream->addr.s;
+ _stream->addr_info.ai_next=NULL;
+ memcpy(&_stream->addr,addrs[pi]->ai_addr,addrs[pi]->ai_addrlen);
+ }
+ if(OP_URL_IS_SSL(&_stream->url)){+ SSL *ssl_conn;
+ /*Start the SSL connection.*/
+ OP_ASSERT(_stream->ssl_ctx!=NULL);
+ ssl_conn=SSL_new(_stream->ssl_ctx);
+ if(OP_LIKELY(ssl_conn!=NULL)){+ ret=op_http_conn_start_tls(_stream,_conn,fds[pi].fd,ssl_conn);
+ if(OP_LIKELY(ret>=0))return ret;
+ SSL_free(ssl_conn);
+ }
+ close(fds[pi].fd);
+ _conn->fd=-1;
+ return OP_FALSE;
+ }
+ /*Just a normal non-SSL connection.*/
+ _conn->ssl_conn=NULL;
+ _conn->fd=fds[pi].fd;
+ _conn->nrequests_left=OP_PIPELINE_MAX_REQUESTS;
+ /*Disable write coalescing.
+ We always send whole requests at once and always parse the response headers
+ before sending another one.*/
+ op_sock_set_tcp_nodelay(fds[pi].fd,1);
+ return 0;
+}
+
+# define OP_BASE64_LENGTH(_len) (((_len)+2)/3*4)
+
+static const char BASE64_TABLE[64]={+ 'A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P',
+ 'Q','R','S','T','U','V','W','X','Y','Z','a','b','c','d','e','f',
+ 'g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v',
+ 'w','x','y','z','0','1','2','3','4','5','6','7','8','9','+','/'
+};
+
+static char *op_base64_encode(char *_dst,const char *_src,int _len){+ unsigned s0;
+ unsigned s1;
+ unsigned s2;
+ int ngroups;
+ int i;
+ ngroups=_len/3;
+ for(i=0;i<ngroups;i++){+ s0=_src[3*i+0];
+ s1=_src[3*i+1];
+ s2=_src[3*i+2];
+ _dst[4*i+0]=BASE64_TABLE[s0>>2];
+ _dst[4*i+1]=BASE64_TABLE[s0&3<<4|s1>>4];
+ _dst[4*i+2]=BASE64_TABLE[s1&15<<2|s2>>6];
+ _dst[4*i+3]=BASE64_TABLE[s2&63];
+ }
+ _len-=3*i;
+ if(_len==1){+ s0=_src[3*i+0];
+ _dst[4*i+0]=BASE64_TABLE[s0>>2];
+ _dst[4*i+1]=BASE64_TABLE[s0&3<<4];
+ _dst[4*i+2]='=';
+ _dst[4*i+3]='=';
+ i++;
+ }
+ else if(_len==2){+ s0=_src[3*i+0];
+ s1=_src[3*i+1];
+ _dst[4*i+0]=BASE64_TABLE[s0>>2];
+ _dst[4*i+1]=BASE64_TABLE[s0&3<<4|s1>>4];
+ _dst[4*i+2]=BASE64_TABLE[s1&15<<2];
+ _dst[4*i+3]='=';
+ i++;
+ }
+ _dst[4*i]='\0';
+ return _dst+4*i;
+}
+
+/*Construct an HTTP authorization header using RFC 2617's Basic Authentication
+ Scheme and append it to the given string buffer.*/
+static int op_sb_append_basic_auth_header(OpusStringBuf *_sb,
+ const char *_header,const char *_user,const char *_pass){+ int user_len;
+ int pass_len;
+ int user_pass_len;
+ int base64_len;
+ int nbuf_total;
+ int ret;
+ ret=op_sb_append_string(_sb,_header);
+ ret|=op_sb_append(_sb,": Basic ",8);
+ user_len=strlen(_user);
+ pass_len=strlen(_pass);
+ if(OP_UNLIKELY(pass_len>INT_MAX-user_len))return OP_EFAULT;
+ if(OP_UNLIKELY(user_len+pass_len>(INT_MAX>>2)*3-3))return OP_EFAULT;
+ user_pass_len=user_len+1+pass_len;
+ base64_len=OP_BASE64_LENGTH(user_pass_len);
+ /*Stick "user:pass" at the end of the buffer so we can Base64 encode it
+ in-place.*/
+ nbuf_total=_sb->nbuf;
+ if(OP_UNLIKELY(base64_len>INT_MAX-nbuf_total))return OP_EFAULT;
+ nbuf_total+=base64_len;
+ ret|=op_sb_ensure_capacity(_sb,nbuf_total);
+ if(OP_UNLIKELY(ret<0))return ret;
+ _sb->nbuf=nbuf_total-user_pass_len;
+ ret=op_sb_append(_sb,_user,user_len);
+ OP_ASSERT(!ret);
+ ret=op_sb_append(_sb,":",1);
+ OP_ASSERT(!ret);
+ ret=op_sb_append(_sb,_pass,pass_len);
+ OP_ASSERT(!ret);
+ op_base64_encode(_sb->buf+nbuf_total-base64_len,
+ _sb->buf+nbuf_total-user_pass_len,user_pass_len);
+ return op_sb_append(_sb,"\r\n",2);
+}
+
+static int op_http_allow_pipelining(const char *_server){+ /*Servers known to do bad things with pipelined requests.
+ This list is taken from Gecko's nsHttpConnection::SupportsPipelining() (in
+ netwerk/protocol/http/nsHttpConnection.cpp).*/
+ static const char *BAD_SERVERS[]={+ "EFAServer/",
+ "Microsoft-IIS/4.",
+ "Microsoft-IIS/5.",
+ "Netscape-Enterprise/3.",
+ "Netscape-Enterprise/4.",
+ "Netscape-Enterprise/5.",
+ "Netscape-Enterprise/6.",
+ "WebLogic 3.",
+ "WebLogic 4.",
+ "WebLogic 5.",
+ "WebLogic 6.",
+ "Winstone Servlet Engine v0."
+ };
+# define NBAD_SERVERS ((int)(sizeof(BAD_SERVERS)/sizeof(*BAD_SERVERS)))
+ if(*_server>='E'&&*_server<='W'){+ int si;
+ for(si=0;si<NBAD_SERVERS;si++){+ if(strncmp(_server,BAD_SERVERS[si],strlen(BAD_SERVERS[si]))==0){+ return 0;
+ }
+ }
+ }
+ return 1;
+# undef NBAD_SERVERS
+}
+
static int op_http_stream_open(OpusHTTPStream *_stream,const char *_url,
int _flags,const char *_proxy_host,unsigned _proxy_port,
const char *_proxy_user,const char *_proxy_pass){@@ -1267,6 +1658,8 @@
char *status_code;
const char *host;
unsigned port;
+ int minor_version_pos;
+ int v1_1_compat;
if(_proxy_host==NULL){host=_stream->url.host;
port=_stream->url.port;
@@ -1291,13 +1684,23 @@
/*Initialize the SSL library if necessary.*/
if(OP_URL_IS_SSL(&_stream->url)&&_stream->ssl_ctx==NULL){SSL_CTX *ssl_ctx;
- /*We need to establish a CONNECT tunnel to handle https proxying.
- This isn't supported yet.*/
- if(_proxy_host!=NULL)return OP_EIMPL;
- /*TODO: The documentation says this is not re-entrant.*/
+# if !defined(OPENSSL_NO_LOCKING)
+ /*The documentation says SSL_library_init() is not reentrant.
+ We don't want to add our own depenencies on a threading library, and it
+ appears that it's safe to call OpenSSL's locking functions before the
+ library is initialized, so that's what we'll do (really OpenSSL should
+ do this for us).
+ This doesn't guarantee that _other_ threads in the application aren't
+ calling SSL_library_init() at the same time, but there's not much we
+ can do about that.*/
+ CRYPTO_w_lock(CRYPTO_LOCK_SSL);
+# endif
SSL_library_init();
/*Needed to get SHA2 algorithms with old OpenSSL versions.*/
OpenSSL_add_ssl_algorithms();
+# if !defined(OPENSSL_NO_LOCKING)
+ CRYPTO_w_unlock(CRYPTO_LOCK_SSL);
+# endif
ssl_ctx=SSL_CTX_new(SSLv23_client_method());
if(ssl_ctx==NULL)return OP_EFAULT;
if(!(_flags&OP_SSL_SKIP_CERTIFICATE_CHECK)){@@ -1304,12 +1707,35 @@
SSL_CTX_set_verify(ssl_ctx,SSL_VERIFY_PEER,NULL);
}
_stream->ssl_ctx=ssl_ctx;
+ if(_proxy_host!=NULL){+ /*We need to establish a CONNECT tunnel to handle https proxying.
+ Build the request we'll send to do so.*/
+ ret=op_sb_append(&_stream->proxy_connect,"CONNECT ",8);
+ ret|=op_sb_append_string(&_stream->proxy_connect,_stream->url.host);
+ ret|=op_sb_append_port(&_stream->proxy_connect,_stream->url.port);
+ /*CONNECT requires at least HTTP 1.1.*/
+ ret|=op_sb_append(&_stream->proxy_connect," HTTP/1.1\r\n",11);
+ ret|=op_sb_append(&_stream->proxy_connect,"Host: ",6);
+ ret|=op_sb_append_string(&_stream->proxy_connect,_stream->url.host);
+ /*The example in RFC 2817 Section 5.2 specifies an explicit port even
+ when connecting to the default port.
+ Given that the proxy doesn't know whether we're trying to connect to
+ an http or an https URL except by the port number, this seems like a
+ good idea.*/
+ ret|=op_sb_append_port(&_stream->proxy_connect,_stream->url.port);
+ ret|=op_sb_append(&_stream->proxy_connect,"\r\n",2);
+ ret|=op_sb_append(&_stream->proxy_connect,"User-Agent: .\r\n",15);
+ if(_proxy_user!=NULL&&_proxy_pass!=NULL){+ ret|=op_sb_append_basic_auth_header(&_stream->proxy_connect,
+ "Proxy-Authorization",_proxy_user,_proxy_pass);
+ }
+ /*For backwards compatibility.*/
+ ret|=op_sb_append(&_stream->proxy_connect,
+ "Proxy-Connection: keep-alive\r\n",30);
+ ret|=op_sb_append(&_stream->proxy_connect,"\r\n",2);
+ if(OP_UNLIKELY(ret<0))return ret;
+ }
}
- /*Pop connection 0 off the free list and put it on the LRU list.*/
- OP_ASSERT(_stream->free_head==_stream->conns+0);
- _stream->lru_head=_stream->conns+0;
- _stream->free_head=_stream->conns[0].next;
- _stream->conns[0].next=NULL;
/*Actually make the connection.*/
if(addrs!=&_stream->addr_info){addrs=op_resolve(host,port);
@@ -1323,14 +1749,22 @@
ret=op_sb_append(&_stream->request,"GET ",4);
ret|=op_sb_append_string(&_stream->request,
_proxy_host!=NULL?_url:_stream->url.path);
+ /*Send HTTP/1.0 by default for maximum compatibility (so we don't have to
+ re-try if HTTP/1.1 fails, though it shouldn't, even for a 1.0 server).
+ This means we aren't conditionally compliant with RFC 2145, because we
+ violate the requirement that "An HTTP client SHOULD send a request
+ version equal to the highest version for which the client is at least
+ conditionally compliant...".
+ According to RFC 2145, that means we can't claim any compliance with any
+ IETF HTTP specification.*/
ret|=op_sb_append(&_stream->request," HTTP/1.0\r\n",11);
+ /*Remember where this is so we can upgrade to HTTP/1.1 if the server
+ supports it.*/
+ minor_version_pos=_stream->request.nbuf-3;
ret|=op_sb_append(&_stream->request,"Host: ",6);
ret|=op_sb_append_string(&_stream->request,_stream->url.host);
if(!OP_URL_IS_DEFAULT_PORT(&_stream->url)){- char port[7];
- OP_ASSERT(_stream->url.port<=65535U);
- sprintf(port,":%u",_stream->url.port);
- ret|=op_sb_append_string(&_stream->request,port);
+ ret|=op_sb_append_port(&_stream->request,_stream->url.port);
}
ret|=op_sb_append(&_stream->request,"\r\n",2);
/*User-Agents have been a bad idea, so send as little as possible.
@@ -1337,7 +1771,8 @@
RFC 2616 requires at least one token in the User-Agent, which must have
at least one character.*/
ret|=op_sb_append(&_stream->request,"User-Agent: .\r\n",15);
- if(_proxy_host!=NULL&&_proxy_user!=NULL&&_proxy_pass!=NULL){+ if(_proxy_host!=NULL&&!OP_URL_IS_SSL(&_stream->url)
+ &&_proxy_user!=NULL&&_proxy_pass!=NULL){ret|=op_sb_append_basic_auth_header(&_stream->request,
"Proxy-Authorization",_proxy_user,_proxy_pass);
}
@@ -1355,7 +1790,9 @@
way to know.*/
/*TODO: Should we update this on redirects?*/
ret|=op_sb_append(&_stream->request,"Referer: /\r\n",12);
- /*Always send a Range request header to find out if we're seekable.*/
+ /*Always send a Range request header to find out if we're seekable.
+ This requires an HTTP/1.1 server to succeed, but we'll still get what we
+ want with an HTTP/1.0 server that ignores this request header.*/
ret|=op_sb_append(&_stream->request,"Range: bytes=0-\r\n",17);
/*Remember where this is so we can append offsets to it later.*/
_stream->request_tail=_stream->request.nbuf-4;
@@ -1369,15 +1806,18 @@
if(OP_UNLIKELY(ret<0))return ret;
ret=ftime(&end_time);
OP_ASSERT(!ret);
- next=op_http_parse_status_line(&status_code,response);
- if(next==NULL)return OP_FALSE;
+ next=op_http_parse_status_line(&v1_1_compat,&status_code,response);
+ if(OP_UNLIKELY(next==NULL))return OP_FALSE;
if(status_code[0]=='2'){opus_int64 content_length;
opus_int64 range_length;
+ int pipeline;
/*We only understand 20x codes.*/
if(status_code[1]!='0')return OP_FALSE;
content_length=-1;
range_length=-1;
+ /*Pipelining is disabled by default.*/
+ pipeline=0;
for(;;){char *header;
char *cdr;
@@ -1386,11 +1826,11 @@
if(header==NULL)break;
if(strcmp(header,"content-length")==0){/*Two Content-Length headers?*/
- if(OP_UNLIKELY(content_length!=-1))return OP_FALSE;
+ if(OP_UNLIKELY(content_length>=0))return OP_FALSE;
content_length=op_http_parse_content_length(cdr);
if(OP_UNLIKELY(content_length<0))return (int)content_length;
/*Make sure the Content-Length and Content-Range headers match.*/
- if(range_length!=-1&&OP_UNLIKELY(content_length!=range_length)){+ if(range_length>=0&&OP_UNLIKELY(content_length!=range_length)){return OP_FALSE;
}
}
@@ -1398,7 +1838,7 @@
opus_int64 range_first;
opus_int64 range_last;
/*Two Content-Range headers?*/
- if(OP_UNLIKELY(range_length!=-1))return OP_FALSE;
+ if(OP_UNLIKELY(range_length>=0))return OP_FALSE;
ret=op_http_parse_content_range(&range_first,&range_last,
&range_length,cdr);
if(OP_UNLIKELY(ret<0))return ret;
@@ -1419,10 +1859,34 @@
/*If there was no length, use the end of the range.*/
else if(range_last>=0)range_length=range_last+1;
/*Make sure the Content-Length and Content-Range headers match.*/
- if(content_length!=-1&&OP_UNLIKELY(content_length!=range_length)){+ if(content_length>=0&&OP_UNLIKELY(content_length!=range_length)){return OP_FALSE;
}
}
+ else if(strcmp(header,"connection")==0){+ /*According to RFC 2616, if an HTTP/1.1 application does not support
+ pipelining, it "MUST include the 'close' connection option in
+ every message."
+ Therefore, if we receive one in the initial response, disable
+ pipelining entirely.
+ The server still might support it (e.g., we might just have hit the
+ request limit for a temporary child process), but if it doesn't
+ and we assume it does, every time we cross a chunk boundary we'll
+ error out and reconnect, adding lots of latency.*/
+ ret=op_http_parse_connection(cdr);
+ if(OP_UNLIKELY(ret<0))return ret;
+ pipeline-=ret;
+ }
+ else if(strcmp(header,"server")){+ /*If we got a Server response header, and it wasn't from a known-bad
+ server, enable pipelining, as long as it's at least HTTP/1.1.
+ According to RFC 2145, the server is supposed to respond with the
+ highest minor version number it supports unless it is known or
+ suspected that we incorrectly implement the HTTP specification.
+ So it should send back at least HTTP/1.1, despite our HTTP/1.0
+ request.*/
+ pipeline+=v1_1_compat&&op_http_allow_pipelining(cdr);
+ }
}
switch(status_code[2]){/*200 OK*/
@@ -1431,7 +1895,7 @@
case '3':break;
/*204 No Content*/
case '4':{- if(content_length!=-1&&OP_UNLIKELY(content_length!=0)){+ if(content_length>=0&&OP_UNLIKELY(content_length!=0)){return OP_FALSE;
}
}break;
@@ -1438,7 +1902,7 @@
/*206 Partial Content*/
case '6':{/*No Content-Range header.*/
- if(OP_UNLIKELY(range_length==-1))return OP_FALSE;
+ if(OP_UNLIKELY(range_length<0))return OP_FALSE;
content_length=range_length;
/*The server supports range requests for this resource.
We can seek.*/
@@ -1456,7 +1920,12 @@
default:return OP_FALSE;
}
_stream->content_length=content_length;
+ _stream->pipeline=pipeline>0;
+ /*Pipelining requires HTTP/1.1 persistent connections.*/
+ if(pipeline)_stream->request.buf[minor_version_pos]='1';
_stream->conns[0].pos=0;
+ _stream->conns[0].end_pos=_stream->seekable?content_length:-1;
+ _stream->conns[0].chunk_size=-1;
_stream->cur_conni=0;
_stream->connect_rate=op_time_diff_ms(&end_time,&start_time);
_stream->connect_rate=OP_MAX(_stream->connect_rate,1);
@@ -1509,42 +1978,95 @@
if(last_host!=_proxy_host)_ogg_free((void *)last_host);
return ret;
}
- op_http_conn_close(_stream,_stream->conns+0);
+ op_http_conn_close(_stream,_stream->conns+0,&_stream->lru_head,1);
}
/*Redirection limit reached.*/
return OP_FALSE;
}
-static int op_http_conn_open_pos(OpusHTTPStream *_stream,
- OpusHTTPConn *_conn,opus_int64 _pos){- struct timeb start_time;
- struct timeb end_time;
- char response[OP_RESPONSE_SIZE_MAX];
- char *next;
- char *status_code;
- opus_int64 range_length;
- opus_int32 connect_rate;
- opus_int32 connect_time;
- int ret;
- ret=op_http_connect(_stream,_conn,&_stream->addr_info,&start_time);
- if(OP_UNLIKELY(ret<0))return ret;
+static int op_http_conn_send_request(OpusHTTPStream *_stream,
+ OpusHTTPConn *_conn,opus_int64 _pos,opus_int32 _chunk_size,
+ int _try_not_to_block){+ opus_int64 next_end;
+ int ret;
+ /*We can't make a new request on a connection that's reading until EOF.*/
+ OP_ASSERT(_conn->end_pos>=0);
+ /*We shouldn't have another request outstanding.*/
+ OP_ASSERT(_conn->next_pos<0);
/*Build the request to send.*/
+ OP_ASSERT(_stream->request.nbuf>=_stream->request_tail);
_stream->request.nbuf=_stream->request_tail;
ret=op_sb_append_nonnegative_int64(&_stream->request,_pos);
- ret|=op_sb_append(&_stream->request,"-\r\n\r\n",5);
+ ret|=op_sb_append(&_stream->request,"-",1);
+ if(_chunk_size>0&&_pos<=OP_INT64_MAX-_chunk_size
+ &&_pos+_chunk_size<_stream->content_length){+ /*We shouldn't be pipelining requests with non-HTTP/1.1 servers.*/
+ OP_ASSERT(_stream->pipeline);
+ next_end=_pos+_chunk_size;
+ ret|=op_sb_append_nonnegative_int64(&_stream->request,next_end-1);
+ /*Use a larger chunk size for our next request.*/
+ _chunk_size<<=1;
+ /*But after a while, just request the rest of the resource.*/
+ if(_chunk_size>OP_PIPELINE_CHUNK_SIZE_MAX)_chunk_size=-1;
+ }
+ else{+ next_end=-1;
+ _chunk_size=-1;
+ }
+ ret|=op_sb_append(&_stream->request,"\r\n\r\n",4);
if(OP_UNLIKELY(ret<0))return ret;
+ /*If we don't want to block, check to see if there's enough space in the send
+ queue.
+ There's still a chance we might block, even if there is enough space, but
+ it's a much slimmer one.
+ Blocking at all is pretty unlikely, as we won't have any requests queued
+ when _try_not_to_block is set, so if FIONSPACE isn't available (e.g., on
+ Linux), just skip the test.*/
+ if(_try_not_to_block){+# if defined(FIONSPACE)
+ int available;
+ ret=ioctl(_conn->fd,FIONSPACE,&available);
+ if(ret<0||available<_stream->request.nbuf)return 1;
+# endif
+ }
ret=op_http_conn_write_fully(_conn,
_stream->request.buf,_stream->request.nbuf);
if(OP_UNLIKELY(ret<0))return ret;
+ _conn->next_pos=_pos;
+ _conn->next_end=next_end;
+ /*Save the chunk size to use for the next request.*/
+ _conn->chunk_size=_chunk_size;
+ _conn->nrequests_left--;
+ return ret;
+}
+
+/*Handles the response to all requests after the first one.
+ Return: 1 if the connection was closed or timed out, 0 on success, or a
+ negative value on any other error.*/
+static int op_http_conn_handle_response(OpusHTTPStream *_stream,
+ OpusHTTPConn *_conn){+ char response[OP_RESPONSE_SIZE_MAX];
+ char *next;
+ char *status_code;
+ opus_int64 range_length;
+ opus_int64 next_pos;
+ opus_int64 next_end;
+ int ret;
ret=op_http_conn_read_response(_conn,
response,sizeof(response)/sizeof(*response));
- if(OP_UNLIKELY(ret<0))return ret;
- ret=ftime(&end_time);
- OP_ASSERT(!ret);
- next=op_http_parse_status_line(&status_code,response);
- if(next==NULL)return OP_FALSE;
- /*We _need_ a 206 Partial Content response.*/
- if(strncmp(status_code,"206",3)!=0)return OP_FALSE;
+ /*If the server just closed the connection on us, we may have just hit a
+ connection re-use limit, so we might want to retry.*/
+ if(OP_UNLIKELY(ret<0))return ret==OP_EREAD?1:ret;
+ next=op_http_parse_status_line(NULL,&status_code,response);
+ if(OP_UNLIKELY(next==NULL))return OP_FALSE;
+ /*We _need_ a 206 Partial Content response.
+ Nothing else will do.*/
+ if(strncmp(status_code,"206",3)!=0){+ /*But on a 408 Request Timeout, we might want to re-try.*/
+ return strncmp(status_code,"408",3)==0?1:OP_FALSE;
+ }
+ next_pos=_conn->next_pos;
+ next_end=_conn->next_end;
range_length=-1;
for(;;){char *header;
@@ -1556,30 +2078,93 @@
opus_int64 range_first;
opus_int64 range_last;
/*Two Content-Range headers?*/
- if(OP_UNLIKELY(range_length!=-1))return OP_FALSE;
+ if(OP_UNLIKELY(range_length>=0))return OP_FALSE;
ret=op_http_parse_content_range(&range_first,&range_last,
&range_length,cdr);
if(OP_UNLIKELY(ret<0))return ret;
- /*"A response with satus code 206 (Partial Content) MUST NOTE
+ /*"A response with satus code 206 (Partial Content) MUST NOT
include a Content-Range field with a byte-range-resp-spec of
'*'."*/
if(OP_UNLIKELY(range_first<0)||OP_UNLIKELY(range_last<0))return OP_FALSE;
+ /*We also don't want range_last to overflow.*/
+ if(OP_UNLIKELY(range_last>=OP_INT64_MAX))return OP_FALSE;
+ range_last++;
/*Quit if we didn't get the offset we asked for.*/
- if(range_first!=_pos)return OP_FALSE;
- /*We asked for the rest of the resource.*/
- if(range_length>=0){- /*Quit if we didn't get it.*/
- if(OP_UNLIKELY(range_last!=range_length-1))return OP_FALSE;
+ if(range_first!=next_pos)return OP_FALSE;
+ if(next_end<0){+ /*We asked for the rest of the resource.*/
+ if(range_length>=0){+ /*Quit if we didn't get it.*/
+ if(OP_UNLIKELY(range_last!=range_length))return OP_FALSE;
+ }
+ /*If there was no length, use the end of the range.*/
+ else range_length=range_last;
+ next_end=range_last;
}
- /*If there was no length, use the end of the range.*/
- else if(range_last>=0)range_length=range_last+1;
+ else{+ if(range_last!=next_end)return OP_FALSE;
+ /*If there was no length, use the larger of the content length or the
+ end of this chunk.*/
+ if(range_length<0){+ range_length=OP_MAX(range_last,_stream->content_length);
+ }
+ }
}
+ else if(strcmp(header,"content-length")==0){+ opus_int64 content_length;
+ /*Validate the Content-Length header, if present, against the request we
+ made.*/
+ content_length=op_http_parse_content_length(cdr);
+ if(OP_UNLIKELY(content_length<0))return (int)content_length;
+ if(next_end<0){+ /*If we haven't seen the Content-Range header yet and we asked for the
+ rest of the resource, set next_end, so we can make sure they match
+ when we do find the Content-Range header.*/
+ if(OP_UNLIKELY(next_pos>OP_INT64_MAX-content_length))return OP_FALSE;
+ next_end=next_pos+content_length;
+ }
+ /*Otherwise, make sure they match now.*/
+ else if(OP_UNLIKELY(next_end-next_pos!=content_length))return OP_FALSE;
+ }
+ else if(strcmp(header,"connection")==0){+ ret=op_http_parse_connection(cdr);
+ if(OP_UNLIKELY(ret<0))return ret;
+ /*If the server told us it was going to close the connection, don't make
+ any more requests.*/
+ if(OP_UNLIKELY(ret>0))_conn->nrequests_left=0;
+ }
}
/*No Content-Range header.*/
- if(OP_UNLIKELY(range_length==-1))return OP_FALSE;
+ if(OP_UNLIKELY(range_length<0))return OP_FALSE;
/*Update the content_length if necessary.*/
_stream->content_length=range_length;
- _conn->pos=_pos;
+ _conn->pos=next_pos;
+ _conn->end_pos=next_end;
+ _conn->next_pos=-1;
+ return 0;
+}
+
+/*Open a new connection that will start reading at byte offset _pos.
+ _pos: The byte offset to start readiny from.
+ _chunk_size: The number of bytes to ask for in the initial request, or -1 to
+ request the rest of the resource.
+ This may be more bytes than remain, in which case it will be
+ converted into a request for the rest.*/
+static int op_http_conn_open_pos(OpusHTTPStream *_stream,
+ OpusHTTPConn *_conn,opus_int64 _pos,opus_int32 _chunk_size){+ struct timeb start_time;
+ struct timeb end_time;
+ opus_int32 connect_rate;
+ opus_int32 connect_time;
+ int ret;
+ ret=op_http_connect(_stream,_conn,&_stream->addr_info,&start_time);
+ if(OP_UNLIKELY(ret<0))return ret;
+ ret=op_http_conn_send_request(_stream,_conn,_pos,_chunk_size,0);
+ if(OP_UNLIKELY(ret<0))return ret;
+ ret=op_http_conn_handle_response(_stream,_conn);
+ if(OP_UNLIKELY(ret!=0))return OP_FALSE;
+ ret=ftime(&end_time);
+ OP_ASSERT(!ret);
_stream->cur_conni=_conn-_stream->conns;
OP_ASSERT(_stream->cur_conni>=0&&_stream->cur_conni<OP_NCONNS_MAX);
/*The connection has been successfully opened.
@@ -1591,6 +2176,129 @@
return 0;
}
+/*Read data from the current response body.
+ If we're pipelining and we get close to the end of this response, queue
+ another request.
+ If we've reached the end of this response body, parse the next response and
+ keep going.
+ [out] _buf: Returns the data read.
+ _size: The size of the buffer.
+ _blocking: Whether or not to block until some data is retrieved.*/
+static ptrdiff_t op_http_conn_read_body(OpusHTTPStream *_stream,
+ OpusHTTPConn *_conn,char *_buf,ptrdiff_t _size,int _blocking){+ opus_int64 pos;
+ opus_int64 end_pos;
+ opus_int64 next_pos;
+ opus_int64 content_length;
+ ptrdiff_t nread;
+ int pipeline;
+ int ret;
+ /*Currently this function can only be called on the LRU head.
+ Otherwise, we'd need a _pnext pointer if we needed to close the connection,
+ and re-opening it would re-organize the lists.*/
+ OP_ASSERT(_stream->lru_head==_conn);
+ /*If we try an empty read, we won't be able to tell if we hit an error.*/
+ OP_ASSERT(_size>0);
+ pos=_conn->pos;
+ end_pos=_conn->end_pos;
+ next_pos=_conn->next_pos;
+ pipeline=_stream->pipeline;
+ content_length=_stream->content_length;
+ if(end_pos>=0){+ /*Have we reached the end of the current response body?*/
+ if(pos>=end_pos){+ OP_ASSERT(content_length>=0);
+ /*If this was the end of the stream, we're done.
+ Also return early if a non-blocking read was requested (regardless of
+ whether we might be able to parse the next response without
+ blocking).*/
+ if(content_length<=end_pos||!_blocking)return 0;
+ /*Otherwise, start on the next response.*/
+ if(next_pos<0){+ /*We haven't issued another request yet.*/
+ if(!pipeline||_conn->nrequests_left<=0){+ /*There are two ways to get here: either the server told us it was
+ going to close the connection after the last request, or we
+ thought we were reading the whole resource, but it grew while we
+ were reading it.
+ The only way the latter could have happened is if content_length
+ changed while seeking.
+ Open a new request to read the rest.*/
+ OP_ASSERT(_stream->seekable);
+ /*Try to open a new connection to read another chunk.*/
+ op_http_conn_close(_stream,_conn,&_stream->lru_head,1);
+ /*If we're not pipelining, we should be requesting the rest.*/
+ OP_ASSERT(pipeline||_conn->chunk_size==-1);
+ ret=op_http_conn_open_pos(_stream,_conn,end_pos,_conn->chunk_size);
+ if(OP_UNLIKELY(ret<0))return 0;
+ }
+ else{+ /*Issue the request now (better late than never).*/
+ ret=op_http_conn_send_request(_stream,_conn,pos,_conn->chunk_size,0);
+ if(OP_UNLIKELY(ret<0))return 0;
+ next_pos=_conn->next_pos;
+ OP_ASSERT(next_pos>=0);
+ }
+ }
+ if(next_pos>=0){+ /*We shouldn't be trying to read past the current request body if we're
+ seeking somewhere else.*/
+ OP_ASSERT(next_pos==end_pos);
+ ret=op_http_conn_handle_response(_stream,_conn);
+ if(OP_UNLIKELY(ret<0))return 0;
+ if(OP_UNLIKELY(ret>0)&&pipeline){+ opus_int64 next_end;
+ next_end=_conn->next_end;
+ /*Our request timed out or the server closed the connection.
+ Try re-connecting.*/
+ op_http_conn_close(_stream,_conn,&_stream->lru_head,1);
+ /*Unless there's a bug, we should be able to convert
+ (next_pos,next_end) into valid (_pos,_chunk_size) parameters.*/
+ OP_ASSERT(next_end<0
+ ||next_end-next_pos>=0&&next_end-next_pos<=0x7FFFFFFF);
+ ret=op_http_conn_open_pos(_stream,_conn,next_pos,
+ next_end<0?-1:(opus_int32)(next_end-next_pos));
+ if(OP_UNLIKELY(ret<0))return 0;
+ }
+ else if(OP_UNLIKELY(ret!=0))return OP_FALSE;
+ }
+ pos=_conn->pos;
+ end_pos=_conn->end_pos;
+ content_length=_stream->content_length;
+ }
+ OP_ASSERT(end_pos>pos);
+ _size=OP_MIN(_size,end_pos-pos);
+ }
+ nread=op_http_conn_read(_conn,_buf,_size,_blocking);
+ pos+=nread;
+ _conn->pos=pos;
+ OP_ASSERT(end_pos<0||content_length>=0);
+ /*TODO: If nrequests_left<=0, we can't make a new request, and there will be
+ a big pause after we hit the end of the chunk while we open a new
+ connection.
+ It would be nice to be able to start that process now, but we have no way
+ to do it in the background without blocking (even if we could start it, we
+ have no guarantee the application will return control to us in a
+ sufficiently timely manner to allow us to complete it, and this is
+ uncommon enough that it's not worth using threads just for this).*/
+ if(end_pos>=0&&end_pos<content_length&&next_pos<0
+ &&pipeline&&OP_LIKELY(_conn->nrequests_left>0)){+ opus_int64 request_thresh;
+ opus_int32 chunk_size;
+ /*Are we getting close to the end of the current response body?
+ If so, we should request more data.*/
+ request_thresh=_stream->connect_rate*_conn->read_rate>>12;
+ /*But don't commit ourselves too quickly.*/
+ chunk_size=_conn->chunk_size;
+ if(chunk_size>=0)request_thresh=OP_MIN(chunk_size>>2,request_thresh);
+ if(end_pos-pos<=request_thresh){+ ret=op_http_conn_send_request(_stream,_conn,end_pos,_conn->chunk_size,1);
+ if(OP_UNLIKELY(ret<0))return 0;
+ }
+ }
+ return nread;
+}
+
static size_t op_http_stream_read(void *_ptr,size_t _size,size_t _nmemb,
void *_stream){OpusHTTPStream *stream;
@@ -1609,7 +2317,7 @@
pos=stream->conns[ci].pos;
size=stream->content_length;
/*Check for EOF.*/
- if(size!=-1){+ if(size>=0){if(pos>=size)return 0;
/*Check for a short read.*/
if(total>size-pos){@@ -1617,7 +2325,10 @@
total=_size*_nmemb;
}
}
- if(_size!=1){+ if(_size==1){+ nread=op_http_conn_read_body(stream,stream->conns+ci,_ptr,total,1);
+ }
+ else{ptrdiff_t n;
nread=0;
/*libopusfile doesn't read multi-byte items, but our abstract stream API
@@ -1629,29 +2340,24 @@
nread_item=0;
do{/*Block on the first item, or if we've gotten a partial item.*/
- n=op_http_conn_read(stream->conns+ci,
+ n=op_http_conn_read_body(stream,stream->conns+ci,
_ptr,_size-nread_item,nread==0||nread_item>0);
- pos+=n;
nread_item+=n;
}
while(n>0&&nread_item<(ptrdiff_t)_size);
/*We can still fail to read a whole item if we encounter an error, or if
we hit EOF and didn't know the stream length.
- TODO: The former is okay, the latter is not.*/
+ TODO: The former is okay, the latter is not, but I don't know how to
+ fix it without buffering arbitrarily large amounts of data.*/
if(nread_item>=(ptrdiff_t)_size)nread++;
total-=_size;
}
while(n>0&&total>0);
}
- else{- nread=op_http_conn_read(stream->conns+ci,_ptr,total,1);
- pos+=nread;
- }
- if(OP_LIKELY(nread>0))stream->conns[ci].pos=pos;
- else{+ if(OP_UNLIKELY(nread<=0)){/*We either hit an error or EOF.
Either way, we're done with this connection.*/
- op_http_conn_close(stream,stream->conns+ci);
+ op_http_conn_close(stream,stream->conns+ci,&stream->lru_head,1);
stream->cur_conni=-1;
stream->pos=pos;
}
@@ -1658,25 +2364,102 @@
return nread;
}
-# define OP_READAHEAD_THRESH_MIN (64*1024)
-/*16 kB is the largest size OpenSSL will return at once.*/
-# define OP_READAHEAD_CHUNK_SIZE (16*1024)
+/*Discard data until we reach the _target position.
+ _just_read_ahead: Whether or not this is a plain fast-forward.
+ If 0, we need to issue a new request for a chunk at _target
+ and discard all the data from our current request(s).
+ Otherwise, we should be able to reach _target without
+ issuing any new requests.
+ _target: The stream position to which to read ahead.*/
+static int op_http_conn_read_ahead(OpusHTTPStream *_stream,
+ OpusHTTPConn *_conn,int _just_read_ahead,opus_int64 _target){+ static char dummy_buf[OP_READAHEAD_CHUNK_SIZE];
+ opus_int64 pos;
+ opus_int64 end_pos;
+ opus_int64 next_pos;
+ opus_int64 next_end;
+ ptrdiff_t nread;
+ int ret;
+ pos=_conn->pos;
+ end_pos=_conn->end_pos;
+ next_pos=_conn->next_pos;
+ next_end=_conn->next_end;
+ if(!_just_read_ahead){+ /*We need to issue a new pipelined request.
+ This is the only case where we allow more than one outstanding request
+ at a time, so we need to reset next_pos (we'll restore it below if we
+ did have an outstanding request).*/
+ OP_ASSERT(_stream->pipeline);
+ _conn->next_pos=-1;
+ ret=op_http_conn_send_request(_stream,_conn,_target,
+ OP_PIPELINE_CHUNK_SIZE,0);
+ if(OP_UNLIKELY(ret<0))return ret;
+ }
+ /*We can reach the target position by reading forward in the current chunk.*/
+ if(_just_read_ahead&&(end_pos<0||_target<end_pos))end_pos=_target;
+ else if(next_pos>=0){+ opus_int64 next_next_pos;
+ opus_int64 next_next_end;
+ /*We already have a request outstanding.
+ Finish off the current chunk.*/
+ while(pos<end_pos){+ nread=op_http_conn_read(_conn,dummy_buf,
+ OP_MIN(end_pos-pos,OP_READAHEAD_CHUNK_SIZE),1);
+ /*We failed to read ahead.*/
+ if(nread<=0)return OP_FALSE;
+ pos+=nread;
+ }
+ OP_ASSERT(pos==end_pos);
+ if(_just_read_ahead){+ next_next_pos=next_next_end=-1;
+ end_pos=_target;
+ }
+ else{+ OP_ASSERT(_conn->next_pos==_target);
+ next_next_pos=_target;
+ next_next_end=_conn->next_end;
+ _conn->next_pos=next_pos;
+ _conn->next_end=next_end;
+ end_pos=next_end;
+ }
+ ret=op_http_conn_handle_response(_stream,_conn);
+ if(OP_UNLIKELY(ret!=0))return OP_FALSE;
+ _conn->next_pos=next_next_pos;
+ _conn->next_end=next_next_end;
+ }
+ while(pos<end_pos){+ nread=op_http_conn_read(_conn,dummy_buf,
+ OP_MIN(end_pos-pos,OP_READAHEAD_CHUNK_SIZE),1);
+ /*We failed to read ahead.*/
+ if(nread<=0)return OP_FALSE;
+ pos+=nread;
+ }
+ OP_ASSERT(pos==end_pos);
+ if(!_just_read_ahead){+ ret=op_http_conn_handle_response(_stream,_conn);
+ if(OP_UNLIKELY(ret!=0))return OP_FALSE;
+ }
+ else _conn->pos=end_pos;
+ OP_ASSERT(_conn->pos==_target);
+ return 0;
+}
static int op_http_stream_seek(void *_stream,opus_int64 _offset,int _whence){struct timeb seek_time;
OpusHTTPStream *stream;
OpusHTTPConn *conn;
- OpusHTTPConn *prev;
OpusHTTPConn **pnext;
- OpusHTTPConn **ppnext;
+ OpusHTTPConn *close_conn;
+ OpusHTTPConn **close_pnext;
opus_int64 content_length;
opus_int64 pos;
+ int pipeline;
int ci;
int ret;
stream=(OpusHTTPStream *)_stream;
if(!stream->seekable)return -1;
- /*If we're seekable, we should have gotten a Content-Length.*/
content_length=stream->content_length;
+ /*If we're seekable, we should have gotten a Content-Length.*/
OP_ASSERT(content_length>=0);
ci=stream->cur_conni;
pos=ci<0?content_length:stream->conns[ci].pos;
@@ -1714,57 +2497,71 @@
stream->pos=pos;
return 0;
}
- ppnext=NULL;
+ close_pnext=NULL;
+ close_conn=NULL;
pnext=&stream->lru_head;
- prev=NULL;
conn=stream->lru_head;
+ pipeline=stream->pipeline;
while(conn!=NULL){opus_int64 conn_pos;
+ opus_int64 end_pos;
opus_int64 read_ahead_thresh;
- /*If this connection has been dormant too long, close it.
- This is to prevent us from hitting server/firewall timeouts.*/
- if(op_time_diff_ms(&seek_time,&conn->read_time)>5*1000){- *pnext=conn->next;
- conn->next=stream->lru_head;
- stream->lru_head=conn;
- op_http_conn_close(stream,conn);
+ int available;
+ int just_read_ahead;
+ /*If this connection has been dormant too long or has made too many
+ requests, close it.
+ This is to prevent us from hitting server limits/firewall timeouts.*/
+ if(op_time_diff_ms(&seek_time,&conn->read_time)>
+ OP_CONNECTION_IDLE_TIMEOUT_MS
+ ||conn->nrequests_left<OP_PIPELINE_MIN_REQUESTS){+ op_http_conn_close(stream,conn,pnext,1);
conn=*pnext;
continue;
}
- /*Dividing by 512 instead of 1000 scales this by nearly 2, biasing towards
- connection re-use (and roughly compensating for the ability of the TCP
- window to open up on long reads).*/
+ /*Dividing by 2048 instead of 1000 scales this by nearly 1/2, biasing away
+ from connection re-use (and roughly compensating for the lag required to
+ reopen the TCP window of a connection that's been idle).
+ There's no overflow checking here, because it's vanishingly unlikely, and
+ all it would do is cause us to make poor decisions.*/
read_ahead_thresh=OP_MAX(OP_READAHEAD_THRESH_MIN,
- stream->connect_rate*conn->read_rate>>9);
+ stream->connect_rate*conn->read_rate>>11);
+ available=op_http_conn_estimate_available(conn);
conn_pos=conn->pos;
- if(pos-read_ahead_thresh<=conn_pos&&conn_pos<=pos){- /*Found a suitable connection to re-use.*/
- *pnext=conn->next;
- conn->next=stream->lru_head;
- stream->lru_head=conn;
- while(conn_pos<pos){- static char dummy_buf[OP_READAHEAD_CHUNK_SIZE];
- ptrdiff_t nread;
- nread=op_http_conn_read(conn,dummy_buf,
- OP_MIN(pos-conn_pos,OP_READAHEAD_CHUNK_SIZE),1);
- if(nread==0)break;
- conn_pos+=nread;
- }
- conn->pos=conn_pos;
- /*We failed to read ahead.*/
- if(conn_pos<pos){- op_http_conn_close(stream,conn);
- /*The connection might have become stale, so keep going.*/
+ end_pos=conn->end_pos;
+ if(conn->next_pos>=0){+ OP_ASSERT(end_pos>=0);
+ OP_ASSERT(conn->next_pos==end_pos);
+ end_pos=conn->next_end;
+ }
+ OP_ASSERT(end_pos<0||conn_pos<=end_pos);
+ /*Can we quickly read ahead without issuing a new request?*/
+ just_read_ahead=conn_pos<=pos&&pos-conn_pos-available<=read_ahead_thresh
+ &&(end_pos<0||pos<end_pos);
+ if(just_read_ahead||pipeline&&end_pos>=0
+ &&end_pos-conn_pos-available<=read_ahead_thresh){+ /*Found a suitable connection to re-use.
+ We always attempt to re-use the first suitable connection we find, even
+ if another one might require less read-ahead, under the assumption
+ more recently used connetions have TCP windows that are open wider.
+ This seems to give a slight performance boost over picking the one with
+ the shortest estimated read-ahead time.*/
+ ret=op_http_conn_read_ahead(stream,conn,just_read_ahead,pos);
+ if(OP_UNLIKELY(ret<0)){+ /*The connection might have become stale, so close it and keep going.*/
+ op_http_conn_close(stream,conn,pnext,1);
conn=*pnext;
continue;
}
/*Sucessfully resurrected this connection.*/
+ *pnext=conn->next;
+ conn->next=stream->lru_head;
+ stream->lru_head=conn;
stream->cur_conni=conn-stream->conns;
return 0;
}
- ppnext=pnext;
+ close_pnext=pnext;
+ close_conn=conn;
pnext=&conn->next;
- prev=conn;
conn=conn->next;
}
/*No suitable connections.
@@ -1771,23 +2568,27 @@
Open a new one.*/
if(stream->free_head==NULL){/*All connections in use.
- Expire the oldest one.*/
- OP_ASSERT(prev!=NULL);
- OP_ASSERT(ppnext!=NULL);
- OP_ASSERT(prev->next==NULL);
- *ppnext=NULL;
- prev->next=stream->lru_head;
- stream->lru_head=prev;
- op_http_conn_close(stream,prev);
+ Expire one of them (we should have already picked which one when scanning
+ the list).*/
+ OP_ASSERT(close_conn!=NULL);
+ OP_ASSERT(close_pnext!=NULL);
+ op_http_conn_close(stream,close_conn,close_pnext,1);
}
OP_ASSERT(stream->free_head!=NULL);
conn=stream->free_head;
- stream->free_head=conn->next;
- conn->next=stream->lru_head;
- stream->lru_head=conn;
- ret=op_http_conn_open_pos(stream,conn,pos);
+ /*If we can pipeline, only request a chunk of data.
+ If we're seeking now, there's a good chance we will want to seek again
+ soon, and this avoids committing this connection to reading the rest of
+ the stream.
+ Particularly with SSL or proxies, issuing a new request on the same
+ connection can be substantially faster than opening a new one.
+ This also limits the amount of data the server will blast at us on this
+ connection if we later seek elsewhere and start reading from a different
+ connection.*/
+ ret=op_http_conn_open_pos(stream,conn,pos,
+ pipeline?OP_PIPELINE_CHUNK_SIZE:-1);
if(OP_UNLIKELY(ret<0)){- op_http_conn_close(stream,conn);
+ op_http_conn_close(stream,conn,&stream->lru_head,1);
return -1;
}
return 0;
@@ -1853,11 +2654,11 @@
return stream;
}
#else
- _flags=_flags;
- _proxy_host=_proxy_host;
- _proxy_port=_proxy_port;
- _proxy_user=_proxy_user;
- _proxy_pass=_proxy_pass;
+ (void)_flags;
+ (void)_proxy_host;
+ (void)_proxy_port;
+ (void)_proxy_user;
+ (void)_proxy_pass;
return NULL;
#endif
}