Merge branch 'thirdparty_dev' into dev

# Conflicts:
#	libs/libs3/inc/request_context.h
#	libs/libs3/src/request.c
#	libs/libs3/src/request_context.c

Source commit: ef7a230cc18d09af038c37f159e4758904ecde33

libs/libs3/GNUmakefile

@@ -142,8 +142,7 @@ ifndef CFLAGS
     endif
 endif
 
-CFLAGS += -Wall -Werror -Wshadow -Wextra \
-		  -Iinc \
+CFLAGS += -Wall -Werror -Wshadow -Wextra -Iinc \
           $(CURL_CFLAGS) $(LIBXML2_CFLAGS) \
           -DLIBS3_VER_MAJOR=\"$(LIBS3_VER_MAJOR)\" \
           -DLIBS3_VER_MINOR=\"$(LIBS3_VER_MINOR)\" \

libs/libs3/inc/libs3.h

@@ -512,7 +512,8 @@ typedef enum
     S3CannedAclPrivate                  = 0, /* private */
     S3CannedAclPublicRead               = 1, /* public-read */
     S3CannedAclPublicReadWrite          = 2, /* public-read-write */
-    S3CannedAclAuthenticatedRead        = 3  /* authenticated-read */
+    S3CannedAclAuthenticatedRead        = 3, /* authenticated-read */
+    S3CannedAclBucketOwnerFullControl   = 4  /* bucket-owner-full-control */
 } S3CannedAcl;
 
 
@@ -1254,6 +1255,28 @@ typedef S3Status (S3MultipartCommitResponseCallback)(const char *location,
                                                      void *callbackData);
 
 
+/**
+ * Mechanism for S3 application to customize each CURL easy request
+ * associated with the given S3 request context.
+ *
+ * This callback can be optinally configured using S3_create_request_context_ex
+ * and will be invoked every time a new CURL request is created in the
+ * context of the given CURLM handle. Invocation will occur after
+ * libs3 has finished configuring its own options of CURL, but before
+ * CURL is started.
+ *
+ * @param curl_multi is the CURLM handle associated with this context.
+ * @param curl_easy is the CURL request being created.
+ * @param setupData is the setupCurlCallbackData parameter passed to
+ *        S3_create_request_context_ex.
+ * @return S3StatusOK to continue processing the request, anything else to
+ *         immediately abort the request and pass this status
+ *         to the S3ResponseCompleteCallback for this request.
+ **/
+typedef S3Status (*S3SetupCurlCallback)(void *curlMulti, void *curlEasy,
+                                        void *setupData);
+
+
 /** **************************************************************************
  * Callback Structures
  ************************************************************************** **/
@@ -1597,6 +1620,51 @@ int S3_status_is_retryable(S3Status status);
 S3Status S3_create_request_context(S3RequestContext **requestContextReturn);
 
 
+/**
+ * Extended version of S3_create_request_context used to create S3RequestContext
+ * for curl_multi_socket_action CURLM handles that will be managed by libs3 user.
+ * This type of handles offer better performance for applications with large
+ * number of simultaneous connections. For details, see MULTI_SOCKET chapter here:
+ * https://curl.haxx.se/libcurl/c/libcurl-multi.html
+ *
+ * In this mode libs3 user will
+ *  - create its own CURLM using curl_multi_init()
+ *  - configure it for its own handlers using
+ *    CURLMOPT_SOCKETFUNCTION/CURLMOPT_TIMERFUNCTION/etc
+ *  - use S3_create_request_context_ex to create S3RequestContext
+ *    for the above CURLM handle
+ *  - start S3 request
+ *  - every time setupCurlCallback is called, will configure new CURL
+ *    object with its own handlers using
+ *    CURLOPT_OPENSOCKETFUNCTION/CURLOPT_CLOSESOCKETFUNCTION/etc
+ *  - the moment libs3 adds CURL object to CURLM handle, curl will start
+ *    communicating directly with libs3 user to drive socket operations,
+ *    where libs3 user will be responsible for calling curl_multi_socket_action
+ *    when necessary.
+ *  - whenever curl_multi_socket_action indicates change in running_handles
+ *    libs3 user should call S3_process_request_context to let libs3 process
+ *    any completed curl transfers and notify back to libs3 user if that was
+ *    the final transfer for a given S3 request.
+ *
+ * @param requestContextReturn returns the newly-created S3RequestContext
+ *        structure, which if successfully returned, must be destroyed via a
+ *        call to S3_destroy_request_context when it is no longer needed.  If
+ *        an error status is returned from this function, then
+ *        requestContextReturn will not have been filled in, and
+ *        S3_destroy_request_context should not be called on it
+ * @param curlMulti is the CURLM handle to be associated with this context.
+ * @param setupCurlCallback is an optional callback routine to be invoked
+ *        by libs3 every time another CURL request is being created for
+ *        use in this context.
+ * @param setupCurlCallbackData is an opaque data to be passed to
+ *        setupCurlCallback.
+ **/
+S3Status S3_create_request_context_ex(S3RequestContext **requestContextReturn,
+                                      void *curlMulti,
+                                      S3SetupCurlCallback setupCurlCallback,
+                                      void *setupCurlCallbackData);
+
+
 /**
  * Destroys an S3RequestContext which was created with
  * S3_create_request_context.  Any requests which are currently being
@@ -1672,6 +1740,26 @@ S3Status S3_runonce_request_context(S3RequestContext *requestContext,
                                     int *requestsRemainingReturn);
 
 
+/**
+ * Extract and finish requests completed by curl multi handle mechanism
+ * in curl_multi_socket_action mode. Should be called by libs3 user when
+ * curl_multi_socket_action indicates a change in running_handles.
+ *
+ * @param requestContext is the S3RequestContext to process
+ * @return One of:
+ *         S3StatusOK if request processing proceeded without error
+ *         S3StatusConnectionFailed if the socket connection to the server
+ *             failed
+ *         S3StatusServerFailedVerification if the SSL certificate of the
+ *             server could not be verified.
+ *         S3StatusInternalError if an internal error prevented the
+ *             S3RequestContext from running one or more requests
+ *         S3StatusOutOfMemory if requests could not be processed due to
+ *             an out of memory error
+ **/
+S3Status S3_process_request_context(S3RequestContext *requestContext);
+
+
 /**
  * This function, in conjunction allows callers to manually manage a set of
  * requests using an S3RequestContext.  This function returns the set of file

libs/libs3/inc/request_context.h

@@ -35,6 +35,14 @@
 
 #include "libs3.h"
 
+
+typedef enum
+{
+    S3CurlModeMultiPerform                                  ,
+    S3CurlModeMultiSocket                                   ,
+} S3CurlMode;
+
+
 struct S3RequestContext
 {
 #ifdef WINSCP
@@ -46,11 +54,15 @@ struct S3RequestContext
     void *responseDataCallbackData;
 #else
     CURLM *curlm;
+    S3CurlMode curl_mode;
     
     int verifyPeerSet;
     long verifyPeer;
 
     struct Request *requests;
+
+    S3SetupCurlCallback setupCurlCallback;
+    void *setupCurlCallbackData;
 #endif
 };
 

libs/libs3/src/multipart.c

@@ -102,6 +102,14 @@ static S3Status initialMultipartXmlCallback(const char *elementPath,
     return S3StatusOK;
 }
 
+static S3Status InitialMultipartPropertiesCallback(const S3ResponseProperties *properties, 
+                                                   void *callbackData) 
+{
+  InitialMultipartData *mdata = (InitialMultipartData *)callbackData;
+  return mdata->handler->responseHandler.propertiesCallback(properties, mdata->userdata);
+}
+
+
 void S3_initiate_multipart(S3BucketContext *bucketContext, const char *key,
                           S3PutProperties *putProperties,
                           S3MultipartInitialHandler *handler,
@@ -137,7 +145,7 @@ void S3_initiate_multipart(S3BucketContext *bucketContext, const char *key,
         0,                                            // startByte
         0,                                            // byteCount
         putProperties,                                // putProperties
-        handler->responseHandler.propertiesCallback,  // propertiesCallback
+        InitialMultipartPropertiesCallback,           // propertiesCallback
         0,                                            // toS3Callback
         0,                                            // toS3CallbackTotalSize
         InitialMultipartCallback,                     // fromS3Callback

libs/libs3/src/request.c

@@ -53,6 +53,10 @@
 #define REQUEST_STACK_SIZE 32
 #define SIGNATURE_SCOPE_SIZE 64
 
+#ifdef WINSCP
+#define SIGNATURE_DEBUG
+#endif
+
 static char userAgentG[USER_AGENT_SIZE];
 
 #ifndef WINSCP
@@ -130,7 +134,7 @@ typedef struct RequestComputedValues
     char rangeHeader[128];
 
     // Authorization header
-    char authorizationHeader[1024];
+    char authorizationHeader[4096];
 
     // Request date stamp
     char requestDateISO8601[64];
@@ -184,15 +188,15 @@ static int neon_header_func(void * userdata, ne_request * NeonRequest, const ne_
     Request *request = (Request *) userdata;
 
     void * cursor = NULL;
-    const char * header_name;
-    const char * header_value;
-    while ((cursor = ne_response_header_iterate(NeonRequest, cursor, &header_name, &header_value)) != NULL)
-    {
-        response_headers_handler_add
+    const char * header_name;
+    const char * header_value;
+    while ((cursor = ne_response_header_iterate(NeonRequest, cursor, &header_name, &header_value)) != NULL)
+    {
+        response_headers_handler_add
             (&(request->responseHeadersHandler), header_name, header_value);
-    }
-
-    return 1;
+    }
+
+    return 1;
 }
 
 
@@ -249,15 +253,15 @@ static int neon_write_func(void * data, const char * buf, size_t len)
 
     // WinSCP (ignore empty responses)
     if (len == 0)
-    {
-        return 0;
+    {
+        return 0;
     }
 
     request_headers_done(request);
 
     if (request->requestContext->responseDataCallback != NULL)
     {
-        request->requestContext->responseDataCallback(buf, len, request->requestContext->responseDataCallbackData);
+        request->requestContext->responseDataCallback(buf, len, request->requestContext->responseDataCallbackData);
     }
 
     if (request->status != S3StatusOK) {
@@ -294,8 +298,9 @@ static S3Status append_amz_header(RequestComputedValues *values,
     values->amzHeaders[values->amzHeadersCount++] = &(values->amzHeadersRaw[rawPos]);
 
     const char *headerStr = headerName;
+    
+    char headerNameWithPrefix[S3_MAX_METADATA_SIZE - sizeof(": v")];
     if (addPrefix) {
-        char headerNameWithPrefix[S3_MAX_METADATA_SIZE - sizeof(": v")];
         snprintf(headerNameWithPrefix, sizeof(headerNameWithPrefix),
                  S3_METADATA_HEADER_NAME_PREFIX "%s", headerName);
         headerStr = headerNameWithPrefix;
@@ -369,6 +374,9 @@ static S3Status compose_amz_headers(const RequestParams *params,
         case S3CannedAclPublicReadWrite:
             cannedAclString = "public-read-write";
             break;
+        case S3CannedAclBucketOwnerFullControl:
+            cannedAclString = "bucket-owner-full-control";
+            break;
         default: // S3CannedAclAuthenticatedRead
             cannedAclString = "authenticated-read";
             break;
@@ -766,13 +774,13 @@ static void canonicalize_signature_headers(RequestComputedValues *values)
 // Canonicalizes the resource into params->canonicalizedResource
 static void canonicalize_resource(const S3BucketContext *context,
                                   const char *urlEncodedKey,
-                                  char *buffer)
+                                  char *buffer, unsigned int buffer_max)
 {
     int len = 0;
 
     *buffer = 0;
 
-#define append(str) len += sprintf(&(buffer[len]), "%s", str)
+#define append(str) len += snprintf(&(buffer[len]), buffer_max - len, "%s", str)
 
     if (context->uriStyle == S3UriStylePath) {
         if (context->bucketName && context->bucketName[0]) {
@@ -790,10 +798,12 @@ static void canonicalize_resource(const S3BucketContext *context,
 #undef append
 }
 
-
-static void sort_query_string(const char *queryString, char *result)
+static void sort_query_string(const char *queryString, char *result,
+                              unsigned int result_size)
 {
+#ifdef SIGNATURE_DEBUG
     ne_debug(NULL, NE_DBG_HTTPBODY, "\n--\nsort_and_urlencode\nqueryString: %s\n", queryString);
+#endif
 
     unsigned int numParams = 1;
     const char *tmp = queryString;
@@ -804,10 +814,10 @@ static void sort_query_string(const char *queryString, char *result)
 
     const char** params = new const char*[numParams]; // WINSCP (heap allocation)
 
-    char * tokenized = new char[strlen(queryString) + 1]; // WINSCP (heap allocation)
-    strncpy(tokenized, queryString, strlen(queryString) + 1);
-
-    char *tok = tokenized;
+    // Where did strdup go?!??
+    int queryStringLen = strlen(queryString);
+    char *tok = (char *) malloc(queryStringLen + 1);
+    strcpy(tok, queryString);
     const char *token = NULL;
     char *save = NULL;
     unsigned int i = 0;
@@ -819,37 +829,47 @@ static void sort_query_string(const char *queryString, char *result)
 
     kv_gnome_sort(params, numParams, '=');
 
+#ifdef SIGNATURE_DEBUG
     for (i = 0; i < numParams; i++) {
         ne_debug(NULL, NE_DBG_HTTPBODY, "%d: %s\n", i, params[i]);
     }
+#endif
 
+    // All params are urlEncoded
+#define append(str) len += snprintf(&(result[len]), result_size - len, "%s", str)
     unsigned int pi = 0;
+    unsigned int len = 0;
     for (; pi < numParams; pi++) {
-        // All params are urlEncoded
-        strncat(result, params[pi], strlen(params[pi]));
-        strncat(result, "&", 1);
+        append(params[pi]);
+        append("&");
+    }
+    // Take off the extra '&'
+    if (len > 0) {
+        result[len - 1] = 0;
     }
-    result[strlen(result) - 1] = '\0';
+#undef append
 
     delete[] params; // WINSCP (heap allocation)
-    delete[] tokenized;
+    free(tok);
 }
 
 
 // Canonicalize the query string part of the request into a buffer
 static void canonicalize_query_string(const char *queryParams,
-                                      const char *subResource, char *buffer)
+                                      const char *subResource,
+                                      char *buffer, unsigned int buffer_size)
 {
     int len = 0;
 
     *buffer = 0;
 
-#define append(str) len += sprintf(&(buffer[len]), "%s", str)
+#define append(str) len += snprintf(&(buffer[len]), buffer_size - len, "%s", str)
 
     if (queryParams && queryParams[0]) {
-        char * sorted = new char[strlen(queryParams) * 2]; // WINSCP (heap allocation)
+        int sortedLen = strlen(queryParams) * 2;
+        char * sorted = new char[sortedLen]; // WINSCP (heap allocation)
         sorted[0] = '\0';
-        sort_query_string(queryParams, sorted);
+        sort_query_string(queryParams, sorted, sortedLen);
         append(sorted);
         delete[] sorted; // WINSCP (heap allocation)
     }
@@ -945,7 +965,9 @@ static S3Status compose_auth_header(const RequestParams *params,
 
     buf_append(canonicalRequest, "%s", values->payloadHash);
 
+#ifdef SIGNATURE_DEBUG
     ne_debug(NULL, NE_DBG_HTTPBODY, "--\nCanonical Request:\n%s\n", canonicalRequest);
+#endif
 
     len = 0;
     unsigned char canonicalRequestHash[S3_SHA256_DIGEST_LENGTH];
@@ -968,17 +990,20 @@ static S3Status compose_auth_header(const RequestParams *params,
     if (params->bucketContext.authRegion) {
         awsRegion = params->bucketContext.authRegion;
     }
-    char scope[SIGNATURE_SCOPE_SIZE + 1];
+    char scope[sizeof(values->requestDateISO8601) + sizeof(awsRegion) +
+               sizeof("//s3/aws4_request") + 1];
     snprintf(scope, sizeof(scope), "%.8s/%s/s3/aws4_request",
              values->requestDateISO8601, awsRegion);
 
-    const int stringToSignLen = 17 + 17 + SIGNATURE_SCOPE_SIZE + 1
-        + strlen(canonicalRequestHashHex); // WINSCP (heap allocation)
+    const int stringToSignLen = 17 + 17 + sizeof(values->requestDateISO8601) +
+        sizeof(scope) + sizeof(canonicalRequestHashHex) + 1; // WINSCP (heap allocation)
     char * stringToSign = new char[stringToSignLen];
     snprintf(stringToSign, stringToSignLen, "AWS4-HMAC-SHA256\n%s\n%s\n%s",
              values->requestDateISO8601, scope, canonicalRequestHashHex);
 
+#ifdef SIGNATURE_DEBUG
     ne_debug(NULL, NE_DBG_HTTPBODY, "--\nString to Sign:\n%s\n", stringToSign);
+#endif
 
     const char *secretAccessKey = params->bucketContext.secretAccessKey;
     const int accessKeyLen = strlen(secretAccessKey) + 5; // WINSCP (heap allocation)
@@ -1040,14 +1065,15 @@ static S3Status compose_auth_header(const RequestParams *params,
              "%s/%.8s/%s/s3/aws4_request", params->bucketContext.accessKeyId,
              values->requestDateISO8601, awsRegion);
 
-    snprintf(
-            values->authorizationHeader,
-            sizeof(values->authorizationHeader),
-            "Authorization: AWS4-HMAC-SHA256 Credential=%s,SignedHeaders=%s,Signature=%s",
-            values->authCredential, values->signedHeaders,
-            values->requestSignatureHex);
+    snprintf(values->authorizationHeader,
+             sizeof(values->authorizationHeader),
+             "Authorization: AWS4-HMAC-SHA256 Credential=%s,SignedHeaders=%s,Signature=%s",
+             values->authCredential, values->signedHeaders,
+             values->requestSignatureHex);
 
+#ifdef SIGNATURE_DEBUG
     ne_debug(NULL, NE_DBG_HTTPBODY, "--\nAuthorization Header:\n%s\n", values->authorizationHeader);
+#endif
 
     return S3StatusOK;
 
@@ -1129,7 +1155,7 @@ int neon_ssl_callback(void * user_data, int failures, const ne_ssl_certificate *
     int result = NE_ERROR;
     if (request->requestContext->sslCallback != NULL)
     {
-        result = request->requestContext->sslCallback(failures, certificate, request->requestContext->sslCallbackData);
+        result = request->requestContext->sslCallback(failures, certificate, request->requestContext->sslCallbackData);
     }
     return result;
 }
@@ -1148,12 +1174,12 @@ static S3Status setup_neon(Request *request,
     if (ne_uri_parse(request->uri, &uri) != 0)
     {
         return S3StatusFailedToInitializeRequest;
-    }
+    }
 
     int port = uri.port;
     if (port == 0)
-    {
-      port = ne_uri_defaultport(uri.scheme);
+    {
+      port = ne_uri_defaultport(uri.scheme);
     }
     request->NeonSession = ne_session_create(uri.scheme, uri.host, port);
     if (request->requestContext->sessionCallback != NULL)
@@ -1185,9 +1211,9 @@ static S3Status setup_neon(Request *request,
     ne_buffer * buf = ne_buffer_create();
     ne_buffer_zappend(buf, uri.path);
     if (uri.query != NULL)
-    {
-        ne_buffer_concat(buf, "?", uri.query, NULL);
-    }
+    {
+        ne_buffer_concat(buf, "?", uri.query, NULL);
+    }
     request->NeonRequest = ne_request_create(request->NeonSession, method, buf->data);
     ne_buffer_destroy(buf);
     ne_uri_free(&uri);
@@ -1224,8 +1250,8 @@ static S3Status setup_neon(Request *request,
 
     if (params->timeoutMs > 0) {
         ne_set_read_timeout(request->NeonSession, params->timeoutMs);
-
-        ne_set_connect_timeout(request->NeonSession, params->timeoutMs);
+
+        ne_set_connect_timeout(request->NeonSession, params->timeoutMs);
     }
 
 #define do_add_header(header) \
@@ -1286,8 +1312,8 @@ static void request_deinitialize(Request *request)
 
 
 static S3Status request_get(const RequestParams *params,
-                            S3RequestContext *context,
                             const RequestComputedValues *values,
+                            S3RequestContext *context, // WINSCP (non-const)
                             Request **reqReturn)
 {
     Request *request = 0;
@@ -1349,6 +1375,17 @@ static S3Status request_get(const RequestParams *params,
         return status;
     }
 
+#ifndef WINSCP
+    if (context && context->setupCurlCallback &&
+        (status = context->setupCurlCallback(
+                context->curlm, request->curl,
+                context->setupCurlCallbackData)) != S3StatusOK) {
+        curl_easy_cleanup(request->curl);
+        free(request);
+        return status;
+    }
+#endif
+
     request->propertiesCallback = params->propertiesCallback;
 
     request->toS3Callback = params->toS3Callback;
@@ -1428,8 +1465,8 @@ S3Status request_api_initialize(const char *userAgentInfo, int flags,
         userAgentInfo = "Unknown";
     }
 
-    char platform[96];
     struct utsname utsn;
+    char platform[sizeof(utsn.sysname) + 1 + sizeof(utsn.machine) + 1];
     if (uname(&utsn)) {
         snprintf(platform, sizeof(platform), "Unknown");
     }
@@ -1480,14 +1517,14 @@ static S3Status setup_request(const RequestParams *params,
     struct tm gmt;
     memset(&gmt, 0, sizeof(gmt));
     gmt.tm_sec=st.wSecond;
-    gmt.tm_min=st.wMinute;
-    gmt.tm_hour=st.wHour;
-    gmt.tm_mday=st.wDay;
-    gmt.tm_mon=st.wMonth-1;
-    gmt.tm_year=(st.wYear>=1900?st.wYear-1900:0);
-    gmt.tm_wday=st.wDayOfWeek;
-    gmt.tm_yday=-1; /* GetSystemTime doesn't tell us */
-    gmt.tm_isdst=0; /* GetSystemTime doesn't tell us */
+    gmt.tm_min=st.wMinute;
+    gmt.tm_hour=st.wHour;
+    gmt.tm_mday=st.wDay;
+    gmt.tm_mon=st.wMonth-1;
+    gmt.tm_year=(st.wYear>=1900?st.wYear-1900:0);
+    gmt.tm_wday=st.wDayOfWeek;
+    gmt.tm_yday=-1; /* GetSystemTime doesn't tell us */
+    gmt.tm_isdst=0; /* GetSystemTime doesn't tell us */
     strftime(computed->requestDateISO8601, sizeof(computed->requestDateISO8601),
              "%Y%m%dT%H%M%SZ", &gmt);
 
@@ -1512,20 +1549,24 @@ static S3Status setup_request(const RequestParams *params,
 
     // Compute the canonicalized resource
     canonicalize_resource(&params->bucketContext, computed->urlEncodedKey,
-                          computed->canonicalURI);
+                          computed->canonicalURI,
+                          sizeof(computed->canonicalURI));
     canonicalize_query_string(params->queryParams, params->subResource,
-                              computed->canonicalQueryString);
+                              computed->canonicalQueryString,
+                              sizeof(computed->canonicalQueryString));
 
     // Compose Authorization header
     if ((status = compose_auth_header(params, computed)) != S3StatusOK) {
         return status;
     }
 
+#ifdef SIGNATURE_DEBUG
     int i = 0;
     ne_debug(NULL, NE_DBG_HTTPBODY, "\n--\nAMZ Headers:\n");
     for (; i < computed->amzHeadersCount; i++) {
         ne_debug(NULL, NE_DBG_HTTPBODY, "%s\n", computed->amzHeaders[i]);
     }
+#endif
 
     return status;
 }
@@ -1547,7 +1588,7 @@ void request_perform(const RequestParams *params, S3RequestContext *context)
     }
 
     // Get an initialized Request structure now
-    if ((status = request_get(params, context /*WINSCP*/, &computed, &request)) != S3StatusOK) {
+    if ((status = request_get(params, &computed, context, &request)) != S3StatusOK) {
         return_status(status);
     }
     // WINSCP (we should always verify the peer)
@@ -1607,8 +1648,8 @@ void request_finish(Request *request, NeonCode code)
         if (request->status == S3StatusOK) {
             if (code != NE_OK) {
                 request->status = request_neon_code_to_status(code);
-                if (request->errorParser.s3ErrorDetails.message == NULL) {
-                    const char * neonError = ne_get_error(request->NeonSession);
+                if (request->errorParser.s3ErrorDetails.message == NULL) {
+                    const char * neonError = ne_get_error(request->NeonSession);
                     int allFit;
                     string_buffer_append(request->statusMessage, neonError, strlen(neonError), allFit);
                     request->errorParser.s3ErrorDetails.message = request->statusMessage;
@@ -1732,12 +1773,16 @@ S3Status S3_generate_authenticated_query_string
     }
 
     // Finally, compose the URI, with params
-    char queryParams[sizeof("X-Amz-Algorithm=AWS4-HMAC-SHA256")
-        + sizeof("&X-Amz-Credential=") + MAX_CREDENTIAL_SIZE
-        + sizeof("&X-Amz-Date=") + 16 + sizeof("&X-Amz-Expires=") + 6
-        + sizeof("&X-Amz-SignedHeaders=") + 128 + sizeof("&X-Amz-Signature=")
-        + sizeof(computed.requestSignatureHex) + 1];
-
+    char queryParams[sizeof("X-Amz-Algorithm=AWS4-HMAC-SHA256") +
+                     sizeof("&X-Amz-Credential=") +
+                     sizeof(computed.authCredential) +
+                     sizeof("&X-Amz-Date=") +
+                     sizeof(computed.requestDateISO8601) +
+                     sizeof("&X-Amz-Expires=") + 64 +
+                     sizeof("&X-Amz-SignedHeaders=") +
+                     sizeof(computed.signedHeaders) +
+                     sizeof("&X-Amz-Signature=") +
+                     sizeof(computed.requestSignatureHex) + 1];
     snprintf(queryParams, sizeof(queryParams),
              "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=%s"
              "&X-Amz-Date=%s&X-Amz-Expires=%d"

libs/libs3/src/request_context.c

@@ -32,6 +32,8 @@
 
 #ifndef WINSCP
 #include <curl/curl.h>
+#else
+#define CURLM void
 #endif
 #include <stdlib.h>
 #include <sys/select.h>
@@ -39,7 +41,10 @@
 #include "request_context.h"
 
 
-S3Status S3_create_request_context(S3RequestContext **requestContextReturn)
+S3Status S3_create_request_context_ex(S3RequestContext **requestContextReturn,
+                                      CURLM *curlm,
+                                      S3SetupCurlCallback setupCurlCallback,
+                                      void *setupCurlCallbackData)
 {
     *requestContextReturn = 
         (S3RequestContext *) malloc(sizeof(S3RequestContext));
@@ -51,20 +56,36 @@ S3Status S3_create_request_context(S3RequestContext **requestContextReturn)
 #ifdef WINSCP
     (*requestContextReturn)->sslCallback = NULL;
 #else
-    if (!((*requestContextReturn)->curlm = curl_multi_init())) {
-        free(*requestContextReturn);
-        return S3StatusOutOfMemory;
+    if (curlm) {
+        (*requestContextReturn)->curlm = curlm;
+        (*requestContextReturn)->curl_mode = S3CurlModeMultiSocket;
+    }
+    else {
+        if (!((*requestContextReturn)->curlm = curl_multi_init())) {
+            free(*requestContextReturn);
+            return S3StatusOutOfMemory;
+        }
+
+        (*requestContextReturn)->curl_mode = S3CurlModeMultiPerform;
     }
 
     (*requestContextReturn)->requests = 0;
     (*requestContextReturn)->verifyPeer = 0;
     (*requestContextReturn)->verifyPeerSet = 0;
+    (*requestContextReturn)->setupCurlCallback = setupCurlCallback;
+    (*requestContextReturn)->setupCurlCallbackData = setupCurlCallbackData;
 #endif
 
     return S3StatusOK;
 }
 
 
+S3Status S3_create_request_context(S3RequestContext **requestContextReturn)
+{
+    return S3_create_request_context_ex(requestContextReturn, NULL, NULL, NULL);
+}
+
+
 void S3_destroy_request_context(S3RequestContext *requestContext)
 {
 #ifndef WINSCP
@@ -81,7 +102,8 @@ void S3_destroy_request_context(S3RequestContext *requestContext)
         r = rNext;
     } while (r != rFirst);
 
-    curl_multi_cleanup(requestContext->curlm);
+    if (requestContext->curl_mode == S3CurlModeMultiPerform)
+        curl_multi_cleanup(requestContext->curlm);
 #endif
 
     free(requestContext);
@@ -151,10 +173,62 @@ S3Status S3_runall_request_context(S3RequestContext *requestContext)
 }
 
 
+static S3Status process_request_context(S3RequestContext *requestContext, int *retry)
+{
+    CURLMsg *msg;
+    int junk;
+
+    *retry = 0;
+
+    while ((msg = curl_multi_info_read(requestContext->curlm, &junk))) {
+        if (msg->msg != CURLMSG_DONE) {
+            return S3StatusInternalError;
+        }
+        Request *request;
+        if (curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE,
+                              (char **) (char *) &request) != CURLE_OK) {
+            return S3StatusInternalError;
+        }
+        // Remove the request from the list of requests
+        if (request->prev == request->next) {
+            // It was the only one on the list
+            requestContext->requests = 0;
+        }
+        else {
+            // It doesn't matter what the order of them are, so just in
+            // case request was at the head of the list, put the one after
+            // request to the head of the list
+            requestContext->requests = request->next;
+            request->prev->next = request->next;
+            request->next->prev = request->prev;
+        }
+        if ((msg->data.result != CURLE_OK) &&
+            (request->status == S3StatusOK)) {
+            request->status = request_curl_code_to_status(
+                msg->data.result);
+        }
+        if (curl_multi_remove_handle(requestContext->curlm,
+                                     msg->easy_handle) != CURLM_OK) {
+            return S3StatusInternalError;
+        }
+        // Finish the request, ensuring that all callbacks have been made,
+        // and also releases the request
+        request_finish(request);
+        // Now, since a callback was made, there may be new requests
+        // queued up to be performed immediately, so do so
+        *retry = 1;
+    }
+
+    return S3StatusOK;
+}
+
+
 S3Status S3_runonce_request_context(S3RequestContext *requestContext, 
                                     int *requestsRemainingReturn)
 {
+    S3Status s3_status;
     CURLMcode status;
+    int retry;
 
     do {
         status = curl_multi_perform(requestContext->curlm,
@@ -170,51 +244,24 @@ S3Status S3_runonce_request_context(S3RequestContext *requestContext,
             return S3StatusInternalError;
         }
 
-        CURLMsg *msg;
-        int junk;
-        while ((msg = curl_multi_info_read(requestContext->curlm, &junk))) {
-            if (msg->msg != CURLMSG_DONE) {
-                return S3StatusInternalError;
-            }
-            Request *request;
-            if (curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, 
-                                  (char **) (char *) &request) != CURLE_OK) {
-                return S3StatusInternalError;
-            }
-            // Remove the request from the list of requests
-            if (request->prev == request->next) {
-                // It was the only one on the list
-                requestContext->requests = 0;
-            }
-            else {
-                // It doesn't matter what the order of them are, so just in
-                // case request was at the head of the list, put the one after
-                // request to the head of the list
-                requestContext->requests = request->next;
-                request->prev->next = request->next;
-                request->next->prev = request->prev;
-            }
-            if ((msg->data.result != CURLE_OK) &&
-                (request->status == S3StatusOK)) {
-                request->status = request_curl_code_to_status
-                    (msg->data.result);
-            }
-            if (curl_multi_remove_handle(requestContext->curlm, 
-                                         msg->easy_handle) != CURLM_OK) {
-                return S3StatusInternalError;
-            }
-            // Finish the request, ensuring that all callbacks have been made,
-            // and also releases the request
-            request_finish(request);
-            // Now, since a callback was made, there may be new requests 
-            // queued up to be performed immediately, so do so
-            status = CURLM_CALL_MULTI_PERFORM;
-        }
-    } while (status == CURLM_CALL_MULTI_PERFORM);
+        s3_status = process_request_context(requestContext, &retry);
+    } while (s3_status == S3StatusOK &&
+             (status == CURLM_CALL_MULTI_PERFORM || retry));
 
-    return S3StatusOK;
+    return s3_status;
 }
 
+
+S3Status S3_process_request_context(S3RequestContext *requestContext)
+{
+    int retry;
+    /* In curl_multi_socket_action mode any new requests created during
+       the following call will have already started associated socket
+       operations, so no need to retry here */
+    return process_request_context(requestContext, &retry);
+}
+
+
 S3Status S3_get_request_context_fdsets(S3RequestContext *requestContext,
                                        fd_set *readFdSet, fd_set *writeFdSet,
                                        fd_set *exceptFdSet, int *maxFd)

libs/libs3/src/response_headers_handler.c

@@ -117,7 +117,8 @@ void response_headers_handler_add(ResponseHeadersHandler *handler,
         string_multibuffer_add(handler->responsePropertyStrings, c, 
                                valuelen, fit);
     }
-    else if (!strncasecmp(header, "ETag", namelen)) {
+    else if ((!strncasecmp(header, "ETag", namelen)) 
+         || (!strncasecmp(header, "Etag", namelen))) { // some servers reply with Etag header
         responseProperties->eTag = 
             string_multibuffer_current(handler->responsePropertyStrings);
         string_multibuffer_add(handler->responsePropertyStrings, c, 

libs/libs3/src/s3.c

@@ -495,6 +495,7 @@ static void growbuffer_read(growbuffer **gb, int amt, int *amtReturn,
             buf->next->prev = buf->prev;
         }
         free(buf);
+        buf = NULL;
     }
 }
 
@@ -2288,6 +2289,9 @@ static void put_object(int argc, char **argv, int optindex,
             else if (!strcmp(val, "public-read-write")) {
                 cannedAcl = S3CannedAclPublicReadWrite;
             }
+            else if (!strcmp(val, "bucket-owner-full-control")) {
+                cannedAcl = S3CannedAclBucketOwnerFullControl;
+            }
             else if (!strcmp(val, "authenticated-read")) {
                 cannedAcl = S3CannedAclAuthenticatedRead;
             }
@@ -2442,6 +2446,7 @@ static void put_object(int argc, char **argv, int optindex,
                         MULTIPART_CHUNK_SIZE);
 
         MultipartPartData partData;
+        memset(&partData, 0, sizeof(MultipartPartData));
         int partContentLength = 0;
 
         S3MultipartInitialHandler handler = {
@@ -2492,10 +2497,11 @@ static void put_object(int argc, char **argv, int optindex,
 upload:
         todoContentLength -= MULTIPART_CHUNK_SIZE * manager.next_etags_pos;
         for (seq = manager.next_etags_pos + 1; seq <= totalSeq; seq++) {
-            memset(&partData, 0, sizeof(MultipartPartData));
             partData.manager = &manager;
             partData.seq = seq;
-            partData.put_object_data = data;
+            if (partData.put_object_data.gb==NULL) {
+              partData.put_object_data = data;
+            }
             partContentLength = ((contentLength > MULTIPART_CHUNK_SIZE) ?
                                  MULTIPART_CHUNK_SIZE : contentLength);
             printf("%s Part Seq %d, length=%d\n", srcSize ? "Copying" : "Sending", seq, partContentLength);