Origin Server returning 200 but Edge Returning 504

Website, Application, Performance DNS & Network
1

When I review the instant logs in real time, I can see that a significant number of the requests are being returned a 504 “Gateway Timeout” error. It seems like as much as 20% of the traffic being returned this way - and this includes Google and Microsoft crawlers.

I understand that there is a default timeout of 100 seconds but none of these requests are taking more than a second and most are 200-300 ms.

However, when I examine the nginx logs on the origin host, nginx reports that it returned a 200 status code for these requests. In fact I cannot find any evidence of the origin server returning a 504 for any requests.

Can anybody help me understand what is going on here? What could be the source of these 504 errors and how can I diagnose this.

4 
116.48.138.188 - - [18/Sep/2023:02:27:17 +0000] "GET /doc/sec-sagimet-biosciences-inc-1400118-ex1026-2023-june-23-19531-4891 HTTP/2.0" 200 411840 "https://www.google.com/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36" rt=0.163 uct="0.001" uht="0.132" urt="0.133" 10.2.0.30:9000


{
    "CacheCacheStatus": "miss",
    "CacheResponseBytes": 1406,
    "CacheTieredFill": false,
    "ClientASN": 4760,
    "ClientCountry": "hk",
    "ClientDeviceType": "desktop",
    "ClientIP": "116.48.138.188",
    "ClientMTLSAuthStatus": "unknown",
    "ClientRequestBytes": 502,
    "ClientRequestHost": "xxx",
    "ClientRequestMethod": "GET",
    "ClientRequestPath": "/doc/sec-sagimet-biosciences-inc-1400118-ex1026-2023-june-23-19531-4891",
    "ClientRequestProtocol": "HTTP/1.1",
    "ClientRequestReferer": "",
    "ClientRequestScheme": "https",
    "ClientRequestSource": "earlyHintsCache",
    "ClientRequestURI": "/doc/sec-sagimet-biosciences-inc-1400118-ex1026-2023-june-23-19531-4891",
    "ClientRequestUserAgent": "nginx-ssl early hints",
    "ClientSSLCipher": "NONE",
    "ClientSSLProtocol": "none",
    "ClientSrcPort": 12534,
    "ClientTCPRTTMs": 0,
    "ClientXRequestedWith": "",
    "Cookies": {},
    "EdgeCFConnectingO2O": false,
    "EdgeColoCode": "HKG",
    "EdgeEndTimestamp": "2023-09-18T07:19:46Z",
    "EdgeResponseBodyBytes": 187,
    "EdgeResponseBytes": 506,
    "EdgeResponseContentType": "text/html",
    "EdgeResponseStatus": 504,
    "EdgeServerIP": "",
    "EdgeStartTimestamp": "2023-09-18T07:19:46Z",
    "EdgeTimeToFirstByteMs": 143,
    "FirewallMatchesActions": [],
    "FirewallMatchesRuleIDs": [],
    "FirewallMatchesSources": [],
    "OriginDNSResponseTimeMs": 0,
    "OriginIP": "",
    "OriginRequestHeaderSendDurationMs": 0,
    "OriginResponseDurationMs": 0,
    "OriginResponseHTTPExpires": "",
    "OriginResponseHTTPLastModified": "",
    "OriginResponseHeaderReceiveDurationMs": 0,
    "OriginResponseStatus": 0,
    "OriginSSLProtocol": "unknown",
    "OriginTCPHandshakeDurationMs": 0,
    "OriginTLSHandshakeDurationMs": 0,
    "ParentRayID": "00",
    "RayID": "8087d29342f31fb8",
    "RequestHeaders": {},
    "ResponseHeaders": {},
    "SecurityLevel": "off",
    "SmartRouteColoID": 0,
    "UpperTierColoID": 0,
    "WorkerStatus": "unknown",
    "WorkerSubrequest": false,
    "WorkerSubrequestCount": 0,
    "sampleInterval": 1
}