// Copyright (c) 2015-2021 MinIO, Inc. // // This file is part of MinIO Object Storage stack // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . package cmd import ( "context" "errors" "fmt" "net/http" "net/textproto" "regexp" "strings" "github.com/minio/madmin-go/v3" "github.com/minio/minio/internal/auth" "github.com/minio/minio/internal/handlers" xhttp "github.com/minio/minio/internal/http" "github.com/minio/minio/internal/logger" "github.com/minio/minio/internal/mcontext" xnet "github.com/minio/pkg/v3/net" "golang.org/x/exp/maps" "golang.org/x/exp/slices" ) const ( copyDirective = "COPY" replaceDirective = "REPLACE" accessDirective = "ACCESS" ) // Parses location constraint from the incoming reader. func parseLocationConstraint(r *http.Request) (location string, s3Error APIErrorCode) { // If the request has no body with content-length set to 0, // we do not have to validate location constraint. Bucket will // be created at default region. locationConstraint := createBucketLocationConfiguration{} err := xmlDecoder(r.Body, &locationConstraint, r.ContentLength) if err != nil && r.ContentLength != 0 { internalLogOnceIf(GlobalContext, err, "location-constraint-xml-parsing") // Treat all other failures as XML parsing errors. return "", ErrMalformedXML } // else for both err as nil or io.EOF location = locationConstraint.Location if location == "" { location = globalSite.Region() } if !isValidLocation(location) { return location, ErrInvalidRegion } return location, ErrNone } // Validates input location is same as configured region // of MinIO server. func isValidLocation(location string) bool { region := globalSite.Region() return region == "" || region == location } // Supported headers that needs to be extracted. var supportedHeaders = []string{ "content-type", "cache-control", "content-language", "content-encoding", "content-disposition", "x-amz-storage-class", xhttp.AmzStorageClass, xhttp.AmzObjectTagging, "expires", xhttp.AmzBucketReplicationStatus, "X-Minio-Replication-Server-Side-Encryption-Sealed-Key", "X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm", "X-Minio-Replication-Server-Side-Encryption-Iv", "X-Minio-Replication-Encrypted-Multipart", "X-Minio-Replication-Actual-Object-Size", // Add more supported headers here. } // mapping of internal headers to allowed replication headers var validSSEReplicationHeaders = map[string]string{ "X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "X-Minio-Replication-Server-Side-Encryption-Sealed-Key", "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm", "X-Minio-Internal-Server-Side-Encryption-Iv": "X-Minio-Replication-Server-Side-Encryption-Iv", "X-Minio-Internal-Encrypted-Multipart": "X-Minio-Replication-Encrypted-Multipart", "X-Minio-Internal-Actual-Object-Size": "X-Minio-Replication-Actual-Object-Size", // Add more supported headers here. } // mapping of replication headers to internal headers var replicationToInternalHeaders = map[string]string{ "X-Minio-Replication-Server-Side-Encryption-Sealed-Key": "X-Minio-Internal-Server-Side-Encryption-Sealed-Key", "X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm": "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm", "X-Minio-Replication-Server-Side-Encryption-Iv": "X-Minio-Internal-Server-Side-Encryption-Iv", "X-Minio-Replication-Encrypted-Multipart": "X-Minio-Internal-Encrypted-Multipart", "X-Minio-Replication-Actual-Object-Size": "X-Minio-Internal-Actual-Object-Size", // Add more supported headers here. } // isDirectiveValid - check if tagging-directive is valid. func isDirectiveValid(v string) bool { // Check if set metadata-directive is valid. return isDirectiveCopy(v) || isDirectiveReplace(v) } // Check if the directive COPY is requested. func isDirectiveCopy(value string) bool { // By default if directive is not set we // treat it as 'COPY' this function returns true. return value == copyDirective || value == "" } // Check if the directive REPLACE is requested. func isDirectiveReplace(value string) bool { return value == replaceDirective } // userMetadataKeyPrefixes contains the prefixes of used-defined metadata keys. // All values stored with a key starting with one of the following prefixes // must be extracted from the header. var userMetadataKeyPrefixes = []string{ "x-amz-meta-", "x-minio-meta-", } // extractMetadataFromReq extracts metadata from HTTP header and HTTP queryString. func extractMetadataFromReq(ctx context.Context, r *http.Request) (metadata map[string]string, err error) { return extractMetadata(ctx, textproto.MIMEHeader(r.Form), textproto.MIMEHeader(r.Header)) } func extractMetadata(ctx context.Context, mimesHeader ...textproto.MIMEHeader) (metadata map[string]string, err error) { metadata = make(map[string]string) for _, hdr := range mimesHeader { // Extract all query values. err = extractMetadataFromMime(ctx, hdr, metadata) if err != nil { return nil, err } } // Set content-type to default value if it is not set. if _, ok := metadata[strings.ToLower(xhttp.ContentType)]; !ok { metadata[strings.ToLower(xhttp.ContentType)] = "binary/octet-stream" } // https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w for k := range metadata { if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) { delete(metadata, k) } } if contentEncoding, ok := metadata[strings.ToLower(xhttp.ContentEncoding)]; ok { contentEncoding = trimAwsChunkedContentEncoding(contentEncoding) if contentEncoding != "" { // Make sure to trim and save the content-encoding // parameter for a streaming signature which is set // to a custom value for example: "aws-chunked,gzip". metadata[strings.ToLower(xhttp.ContentEncoding)] = contentEncoding } else { // Trimmed content encoding is empty when the header // value is set to "aws-chunked" only. // Make sure to delete the content-encoding parameter // for a streaming signature which is set to value // for example: "aws-chunked" delete(metadata, strings.ToLower(xhttp.ContentEncoding)) } } // Success. return metadata, nil } // extractMetadata extracts metadata from map values. func extractMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[string]string) error { if v == nil { bugLogIf(ctx, errInvalidArgument) return errInvalidArgument } nv := make(textproto.MIMEHeader, len(v)) for k, kv := range v { // Canonicalize all headers, to remove any duplicates. nv[http.CanonicalHeaderKey(k)] = kv } // Save all supported headers. for _, supportedHeader := range supportedHeaders { value, ok := nv[http.CanonicalHeaderKey(supportedHeader)] if ok { if slices.Contains(maps.Keys(replicationToInternalHeaders), supportedHeader) { m[replicationToInternalHeaders[supportedHeader]] = strings.Join(value, ",") } else { m[supportedHeader] = strings.Join(value, ",") } } } for key := range v { for _, prefix := range userMetadataKeyPrefixes { if !stringsHasPrefixFold(key, prefix) { continue } value, ok := nv[http.CanonicalHeaderKey(key)] if ok { m[key] = strings.Join(value, ",") break } } } return nil } // Returns access credentials in the request Authorization header. func getReqAccessCred(r *http.Request, region string) (cred auth.Credentials) { cred, _, _ = getReqAccessKeyV4(r, region, serviceS3) if cred.AccessKey == "" { cred, _, _ = getReqAccessKeyV2(r) } return cred } // Extract request params to be sent with event notification. func extractReqParams(r *http.Request) map[string]string { if r == nil { return nil } region := globalSite.Region() cred := getReqAccessCred(r, region) principalID := cred.AccessKey if cred.ParentUser != "" { principalID = cred.ParentUser } // Success. m := map[string]string{ "region": region, "principalId": principalID, "sourceIPAddress": handlers.GetSourceIP(r), // Add more fields here. } if rangeField := r.Header.Get(xhttp.Range); rangeField != "" { m["range"] = rangeField } if _, ok := r.Header[xhttp.MinIOSourceReplicationRequest]; ok { m[xhttp.MinIOSourceReplicationRequest] = "" } return m } // Extract response elements to be sent with event notification. func extractRespElements(w http.ResponseWriter) map[string]string { if w == nil { return map[string]string{} } return map[string]string{ "requestId": w.Header().Get(xhttp.AmzRequestID), "nodeId": w.Header().Get(xhttp.AmzRequestHostID), "content-length": w.Header().Get(xhttp.ContentLength), // Add more fields here. } } // Trims away `aws-chunked` from the content-encoding header if present. // Streaming signature clients can have custom content-encoding such as // `aws-chunked,gzip` here we need to only save `gzip`. // For more refer http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html func trimAwsChunkedContentEncoding(contentEnc string) (trimmedContentEnc string) { if contentEnc == "" { return contentEnc } var newEncs []string for _, enc := range strings.Split(contentEnc, ",") { if enc != streamingContentEncoding { newEncs = append(newEncs, enc) } } return strings.Join(newEncs, ",") } func collectInternodeStats(f http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { f.ServeHTTP(w, r) tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt) if !ok || tc == nil { return } globalConnStats.incInternodeInputBytes(int64(tc.RequestRecorder.Size())) globalConnStats.incInternodeOutputBytes(int64(tc.ResponseRecorder.Size())) } } func collectAPIStats(api string, f http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { resource, err := getResource(r.URL.Path, r.Host, globalDomainNames) if err != nil { defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r)) apiErr := errorCodes.ToAPIErr(ErrUnsupportedHostHeader) apiErr.Description = fmt.Sprintf("%s: %v", apiErr.Description, err) writeErrorResponse(r.Context(), w, apiErr, r.URL) return } bucket, _ := path2BucketObject(resource) meta, err := globalBucketMetadataSys.Get(bucket) // check if this bucket exists. countBktStat := bucket != "" && bucket != minioReservedBucket && err == nil && !meta.Created.IsZero() if countBktStat { globalBucketHTTPStats.updateHTTPStats(bucket, api, nil) } globalHTTPStats.currentS3Requests.Inc(api) f.ServeHTTP(w, r) globalHTTPStats.currentS3Requests.Dec(api) tc, _ := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt) if tc != nil { globalHTTPStats.updateStats(api, tc.ResponseRecorder) globalConnStats.incS3InputBytes(int64(tc.RequestRecorder.Size())) globalConnStats.incS3OutputBytes(int64(tc.ResponseRecorder.Size())) if countBktStat { globalBucketConnStats.incS3InputBytes(bucket, int64(tc.RequestRecorder.Size())) globalBucketConnStats.incS3OutputBytes(bucket, int64(tc.ResponseRecorder.Size())) globalBucketHTTPStats.updateHTTPStats(bucket, api, tc.ResponseRecorder) } } } } // Returns "/bucketName/objectName" for path-style or virtual-host-style requests. func getResource(path string, host string, domains []string) (string, error) { if len(domains) == 0 { return path, nil } // If virtual-host-style is enabled construct the "resource" properly. xhost, err := xnet.ParseHost(host) if err != nil { return "", err } for _, domain := range domains { if xhost.Name == minioReservedBucket+"."+domain { continue } if !strings.HasSuffix(xhost.Name, "."+domain) { continue } bucket := strings.TrimSuffix(xhost.Name, "."+domain) return SlashSeparator + pathJoin(bucket, path), nil } return path, nil } var regexVersion = regexp.MustCompile(`^/minio.*/(v\d+)/.*`) func extractAPIVersion(r *http.Request) string { if matches := regexVersion.FindStringSubmatch(r.URL.Path); len(matches) > 1 { return matches[1] } return "unknown" } func methodNotAllowedHandler(api string) func(w http.ResponseWriter, r *http.Request) { return errorResponseHandler } // If none of the http routes match respond with appropriate errors func errorResponseHandler(w http.ResponseWriter, r *http.Request) { if r.Method == http.MethodOptions { return } desc := "Do not upgrade one server at a time - please follow the recommended guidelines mentioned here https://github.com/minio/minio#upgrading-minio for your environment" switch { case strings.HasPrefix(r.URL.Path, peerRESTPrefix): writeErrorResponseString(r.Context(), w, APIError{ Code: "XMinioPeerVersionMismatch", Description: desc, HTTPStatusCode: http.StatusUpgradeRequired, }, r.URL) case strings.HasPrefix(r.URL.Path, storageRESTPrefix): writeErrorResponseString(r.Context(), w, APIError{ Code: "XMinioStorageVersionMismatch", Description: desc, HTTPStatusCode: http.StatusUpgradeRequired, }, r.URL) case strings.HasPrefix(r.URL.Path, adminPathPrefix): var desc string version := extractAPIVersion(r) switch version { case "v1", madmin.AdminAPIVersionV2: desc = fmt.Sprintf("Server expects client requests with 'admin' API version '%s', found '%s', please upgrade the client to latest releases", madmin.AdminAPIVersion, version) case madmin.AdminAPIVersion: desc = fmt.Sprintf("This 'admin' API is not supported by server in '%s'", getMinioMode()) default: desc = fmt.Sprintf("Unexpected client 'admin' API version found '%s', expected '%s', please downgrade the client to older releases", version, madmin.AdminAPIVersion) } writeErrorResponseJSON(r.Context(), w, APIError{ Code: "XMinioAdminVersionMismatch", Description: desc, HTTPStatusCode: http.StatusUpgradeRequired, }, r.URL) default: writeErrorResponse(r.Context(), w, APIError{ Code: "BadRequest", Description: fmt.Sprintf("An error occurred when parsing the HTTP request %s at '%s'", r.Method, r.URL.Path), HTTPStatusCode: http.StatusBadRequest, }, r.URL) } } // gets host name for current node func getHostName(r *http.Request) (hostName string) { if globalIsDistErasure { hostName = globalLocalNodeName } else { hostName = r.Host } return } // Proxy any request to an endpoint. func proxyRequest(ctx context.Context, w http.ResponseWriter, r *http.Request, ep ProxyEndpoint) (success bool) { success = true // Make sure we remove any existing headers before // proxying the request to another node. for k := range w.Header() { w.Header().Del(k) } f := handlers.NewForwarder(&handlers.Forwarder{ PassHost: true, RoundTripper: ep.Transport, ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) { success = false if err != nil && !errors.Is(err, context.Canceled) { replLogIf(GlobalContext, err) } }, }) r.URL.Scheme = "http" if globalIsTLS { r.URL.Scheme = "https" } r.URL.Host = ep.Host f.ServeHTTP(w, r) return }