From 346a9b24fa941a1872f176f2f8310d8235947e26 Mon Sep 17 00:00:00 2001 From: Kenji Okimoto Date: Thu, 22 Nov 2018 11:54:25 +0900 Subject: [PATCH 1/4] Support setting log level per plugin --- lib/fluent/plugin/filter_add_insert_ids.rb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/fluent/plugin/filter_add_insert_ids.rb b/lib/fluent/plugin/filter_add_insert_ids.rb index 7de56623..a1d37d8a 100644 --- a/lib/fluent/plugin/filter_add_insert_ids.rb +++ b/lib/fluent/plugin/filter_add_insert_ids.rb @@ -56,13 +56,12 @@ module ConfigConstants def start super - @log = $log # rubocop:disable Style/GlobalVars # Initialize the insertID. - @log.info "Started the add_insert_ids plugin with #{@insert_id_key}" \ - ' as the insert ID key.' + log.info "Started the add_insert_ids plugin with #{@insert_id_key}" \ + ' as the insert ID key.' @insert_id = generate_initial_insert_id - @log.info "Initialized the insert ID key to #{@insert_id}." + log.info "Initialized the insert ID key to #{@insert_id}." end def configure(conf) From 63387b523f3ee703a2ed0cf702d14b87891500cc Mon Sep 17 00:00:00 2001 From: Kenji Okimoto Date: Thu, 22 Nov 2018 13:58:25 +0900 Subject: [PATCH 2/4] Support log level per plugin We must use `$log` for logging while configuring. --- .rubocop.yml | 5 + lib/fluent/plugin/out_google_cloud.rb | 140 ++++++++++++-------------- 2 files changed, 72 insertions(+), 73 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index 268a8f4e..2492880e 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -5,3 +5,8 @@ AllCops: Exclude: # Generated files. - 'lib/google/**/*' + +Style/GlobalVars: + AllowedVariables: + - $log + diff --git a/lib/fluent/plugin/out_google_cloud.rb b/lib/fluent/plugin/out_google_cloud.rb index 756071f0..5263929d 100644 --- a/lib/fluent/plugin/out_google_cloud.rb +++ b/lib/fluent/plugin/out_google_cloud.rb @@ -441,19 +441,13 @@ module InternalConstants attr_reader :resource attr_reader :common_labels - def initialize - super - # use the global logger - @log = $log # rubocop:disable Style/GlobalVars - end - def configure(conf) super # TODO(qingling128): Remove this warning after the support is added. Also # remove the comment in the description of this configuration. unless @logging_api_url == DEFAULT_LOGGING_API_URL || @use_grpc - @log.warn 'Detected customized logging_api_url while use_grpc is not' \ + $log.warn 'Detected customized logging_api_url while use_grpc is not' \ ' enabled. Customized logging_api_url for the non-gRPC path' \ ' is not supported. The logging_api_url option will be' \ ' ignored.' @@ -571,7 +565,7 @@ def configure(conf) if [Platform::GCE, Platform::EC2].include?(@platform) # Log an informational message containing the Logs viewer URL - @log.info 'Logs viewer address: https://console.cloud.google.com/logs/', + $log.info 'Logs viewer address: https://console.cloud.google.com/logs/', "viewer?project=#{@project_id}&resource=#{@resource.type}/", "instance_id/#{@vm_id}" end @@ -775,7 +769,7 @@ def write_request_via_grpc(entries:, # aid with verification and troubleshooting. unless @successful_call @successful_call = true - @log.info 'Successfully sent gRPC to Stackdriver Logging API.' + log.info 'Successfully sent gRPC to Stackdriver Logging API.' end rescue Google::Gax::GaxError => gax_error @@ -796,8 +790,8 @@ def write_request_via_grpc(entries:, # HTTP status 504 (Gateway Timeout). GRPC::DeadlineExceeded increment_retried_entries_count(entries_count, error.code) - @log.debug "Retrying #{entries_count} log message(s) later.", - error: error.to_s, error_code: error.code.to_s + log.debug "Retrying #{entries_count} log message(s) later.", + error: error.to_s, error_code: error.code.to_s raise error # Most client errors indicate a problem with the request itself and @@ -824,8 +818,8 @@ def write_request_via_grpc(entries:, GRPC::Unknown increment_failed_requests_count(error.code) increment_dropped_entries_count(entries_count, error.code) - @log.warn "Dropping #{entries_count} log message(s)", - error: error.to_s, error_code: error.code.to_s + log.warn "Dropping #{entries_count} log message(s)", + error: error.to_s, error_code: error.code.to_s # If partial_success is enabled, valid entries should have be # written even if some other entries fail due to InvalidArgument or @@ -839,16 +833,16 @@ def write_request_via_grpc(entries:, if error_details_map.empty? increment_failed_requests_count(error.code) increment_dropped_entries_count(entries_count, error.code) - @log.warn "Dropping #{entries_count} log message(s)", - error: error.to_s, error_code: error.code.to_s + log.warn "Dropping #{entries_count} log message(s)", + error: error.to_s, error_code: error.code.to_s else error_details_map.each do |(error_code, error_message), indexes| partial_errors_count = indexes.length increment_dropped_entries_count(partial_errors_count, error_code) entries_count -= partial_errors_count - @log.warn "Dropping #{partial_errors_count} log message(s)", - error: error_message, error_code: error_code.to_s + log.warn "Dropping #{partial_errors_count} log message(s)", + error: error_message, error_code: error_code.to_s end # Consider partially successful requests successful. increment_successful_requests_count @@ -864,9 +858,9 @@ def write_request_via_grpc(entries:, end increment_failed_requests_count(error_code) increment_dropped_entries_count(entries_count, error_code) - @log.error "Unknown response code #{error_code} from the server," \ - " dropping #{entries_count} log message(s)", - error: error.to_s, error_code: error_code.to_s + log.error "Unknown response code #{error_code} from the server," \ + " dropping #{entries_count} log message(s)", + error: error.to_s, error_code: error_code.to_s end # Got an unexpected error (not Google::Gax::GaxError) from the @@ -875,9 +869,9 @@ def write_request_via_grpc(entries:, increment_failed_requests_count(GRPC::Core::StatusCodes::UNKNOWN) increment_dropped_entries_count(entries_count, GRPC::Core::StatusCodes::UNKNOWN) - @log.error "Unexpected error type #{error.class.name} from the client" \ - " library, dropping #{entries_count} log message(s)", - error: error.to_s + log.error "Unexpected error type #{error.class.name} from the client" \ + " library, dropping #{entries_count} log message(s)", + error: error.to_s end def write_request_via_rest(entries:, @@ -903,14 +897,14 @@ def write_request_via_rest(entries:, # with verification and troubleshooting. unless @successful_call @successful_call = true - @log.info 'Successfully sent to Stackdriver Logging API.' + log.info 'Successfully sent to Stackdriver Logging API.' end rescue Google::Apis::ServerError => error # 5xx server errors. Retry via re-raising the error. increment_retried_entries_count(entries_count, error.status_code) - @log.debug "Retrying #{entries_count} log message(s) later.", - error: error.to_s, error_code: error.status_code.to_s + log.debug "Retrying #{entries_count} log message(s) later.", + error: error.to_s, error_code: error.status_code.to_s raise error rescue Google::Apis::AuthorizationError => error @@ -919,8 +913,8 @@ def write_request_via_rest(entries:, # the permissions on the Google Cloud project. increment_failed_requests_count(error.status_code) increment_dropped_entries_count(entries_count, error.status_code) - @log.warn "Dropping #{entries_count} log message(s)", - error: error.to_s, error_code: error.status_code.to_s + log.warn "Dropping #{entries_count} log message(s)", + error: error.to_s, error_code: error.status_code.to_s rescue Google::Apis::ClientError => error # 4xx client errors. Most client errors indicate a problem with the @@ -929,16 +923,16 @@ def write_request_via_rest(entries:, if error_details_map.empty? increment_failed_requests_count(error.status_code) increment_dropped_entries_count(entries_count, error.status_code) - @log.warn "Dropping #{entries_count} log message(s)", - error: error.to_s, error_code: error.status_code.to_s + log.warn "Dropping #{entries_count} log message(s)", + error: error.to_s, error_code: error.status_code.to_s else error_details_map.each do |(error_code, error_message), indexes| partial_errors_count = indexes.length increment_dropped_entries_count(partial_errors_count, error_code) entries_count -= partial_errors_count - @log.warn "Dropping #{partial_errors_count} log message(s)", - error: error_message, - error_code: "google.rpc.Code[#{error_code}]" + log.warn "Dropping #{partial_errors_count} log message(s)", + error: error_message, + error_code: "google.rpc.Code[#{error_code}]" end # Consider partially successful requests successful. increment_successful_requests_count @@ -980,26 +974,26 @@ module Platform # service (unless the user has explicitly disabled using that). def detect_platform unless @use_metadata_service - @log.info 'use_metadata_service is false; not detecting platform' + $log.info 'use_metadata_service is false; not detecting platform' return Platform::OTHER end begin open('http://' + METADATA_SERVICE_ADDR) do |f| if f.meta['metadata-flavor'] == 'Google' - @log.info 'Detected GCE platform' + $log.info 'Detected GCE platform' return Platform::GCE end if f.meta['server'] == 'EC2ws' - @log.info 'Detected EC2 platform' + $log.info 'Detected EC2 platform' return Platform::EC2 end end rescue StandardError => e - @log.error 'Failed to access metadata service: ', error: e + $log.error 'Failed to access metadata service: ', error: e end - @log.info 'Unable to determine platform' + $log.info 'Unable to determine platform' Platform::OTHER end @@ -1079,7 +1073,7 @@ def set_vm_id @vm_id ||= fetch_gce_metadata('instance/id') if @platform == Platform::GCE @vm_id ||= ec2_metadata['instanceId'] if @platform == Platform::EC2 rescue StandardError => e - @log.error 'Failed to obtain vm_id: ', error: e + log.error 'Failed to obtain vm_id: ', error: e end # 1. Return the value if it is explicitly set in the config already. @@ -1087,7 +1081,7 @@ def set_vm_id def set_vm_name @vm_name ||= Socket.gethostname rescue StandardError => e - @log.error 'Failed to obtain vm name: ', error: e + log.error 'Failed to obtain vm name: ', error: e end # 1. Return the value if it is explicitly set in the config already. @@ -1104,7 +1098,7 @@ def set_location @zone ||= 'aws:' + ec2_metadata[aws_location_key] if @platform == Platform::EC2 && ec2_metadata.key?(aws_location_key) rescue StandardError => e - @log.error 'Failed to obtain location: ', error: e + log.error 'Failed to obtain location: ', error: e end # Retrieve monitored resource via the legacy way. @@ -1143,7 +1137,7 @@ def determine_agent_level_monitored_resource_type return resource_type if attributes.superset?(expected) end rescue StandardError => e - @log.error 'Failed to detect subservice: ', error: e + log.error 'Failed to detect subservice: ', error: e end end @@ -1205,8 +1199,8 @@ def determine_agent_level_monitored_resource_labels(type) {} rescue StandardError => e - @log.error "Failed to set monitored resource labels for #{type}: ", - error: e + log.error "Failed to set monitored resource labels for #{type}: ", + error: e {} end @@ -1247,14 +1241,14 @@ def group_log_entries_by_tag_and_local_resource_id(chunk) groups = {} chunk.msgpack_each do |tag, time, record| unless record.is_a?(Hash) - @log.warn 'Dropping log entries with malformed record: ' \ + log.warn 'Dropping log entries with malformed record: ' \ "'#{record.inspect}'. " \ 'A log record should be in JSON format.' next end sanitized_tag = sanitize_tag(tag) if sanitized_tag.nil? - @log.warn "Dropping log entries with invalid tag: '#{tag.inspect}'." \ + log.warn "Dropping log entries with invalid tag: '#{tag.inspect}'." \ ' A tag should be a string with utf8 characters.' next end @@ -1384,11 +1378,11 @@ def determine_group_level_monitored_resource_and_labels(tag, def monitored_resource_from_local_resource_id(local_resource_id) return unless local_resource_id if @enable_metadata_agent - @log.debug 'Calling metadata agent with local_resource_id: ' \ + log.debug 'Calling metadata agent with local_resource_id: ' \ "#{local_resource_id}." resource = query_metadata_agent_for_monitored_resource( local_resource_id) - @log.debug 'Retrieved monitored resource from metadata agent: ' \ + log.debug 'Retrieved monitored resource from metadata agent: ' \ "#{resource.inspect}." if resource # TODO(qingling128): Fix this temporary renaming from 'gke_container' @@ -1400,7 +1394,7 @@ def monitored_resource_from_local_resource_id(local_resource_id) # Fall back to constructing monitored resource locally. # TODO(qingling128): This entire else clause is temporary until we # implement buffering and caching. - @log.debug('Failed to retrieve monitored resource from Metadata' \ + log.debug('Failed to retrieve monitored resource from Metadata' \ " Agent with local_resource_id #{local_resource_id}.") construct_k8s_resource_locally(local_resource_id) end @@ -1487,7 +1481,7 @@ def query_metadata_agent_for_monitored_resource(local_resource_id) begin resource = Google::Api::MonitoredResource.decode_json(response.to_json) rescue Google::Protobuf::ParseError, ArgumentError => e - @log.error 'Error parsing monitored resource from Metadata Agent. ' \ + log.error 'Error parsing monitored resource from Metadata Agent. ' \ "response: #{response.inspect}", error: e return nil end @@ -1505,20 +1499,20 @@ def query_metadata_agent_for_monitored_resource(local_resource_id) # to JSON. Return nil in case of failure. def query_metadata_agent(path) url = "#{@metadata_agent_url}/#{path}" - @log.debug("Calling Metadata Agent: #{url}") + log.debug("Calling Metadata Agent: #{url}") open(url) do |f| response = f.read parsed_hash = parse_json_or_nil(response) if parsed_hash.nil? - @log.error 'Response from Metadata Agent is not in valid json ' \ + log.error 'Response from Metadata Agent is not in valid json ' \ "format: '#{response.inspect}'." return nil end - @log.debug "Response from Metadata Agent: #{parsed_hash}" + log.debug "Response from Metadata Agent: #{parsed_hash}" return parsed_hash end rescue StandardError => e - @log.error "Error calling Metadata Agent at #{url}.", error: e + log.error "Error calling Metadata Agent at #{url}.", error: e nil end @@ -1590,7 +1584,7 @@ def compute_timestamp(resource_type, record, time) unless @timenanos_warning # Warn the user this is deprecated, but only once to avoid spam. @timenanos_warning = true - @log.warn 'timeNanos is deprecated - please use ' \ + log.warn 'timeNanos is deprecated - please use ' \ 'timestampSeconds and timestampNanos instead.' end timestamp = time_or_nil(ts_secs, ts_nanos) @@ -1689,7 +1683,7 @@ def set_log_entry_fields(record, entry) begin casted_value = send(cast_fn, value) rescue TypeError - @log.error "Failed to #{cast_fn} for #{field_name}." \ + log.error "Failed to #{cast_fn} for #{field_name}." \ "#{original_key} with value #{value.inspect}.", err next end @@ -1712,7 +1706,7 @@ def set_log_entry_fields(record, entry) entry.send("#{field_name}=", output) rescue StandardError => err - @log.error "Failed to set log entry field for #{field_name}.", err + log.error "Failed to set log entry field for #{field_name}.", err end end end @@ -1917,7 +1911,7 @@ def value_from_ruby(value) when Array ret.list_value = list_from_ruby(value) else - @log.error "Unknown type: #{value.class}" + log.error "Unknown type: #{value.class}" raise Google::Protobuf::Error, "Unknown type: #{value.class}" end ret @@ -2066,10 +2060,10 @@ def convert_to_utf8(input) begin input.encode('utf-8') rescue EncodingError - @log.error 'Encountered encoding issues potentially due to non ' \ - 'UTF-8 characters. To allow non-UTF-8 characters and ' \ - 'replace them with spaces, please set "coerce_to_utf8" ' \ - 'to true.' + log.error 'Encountered encoding issues potentially due to non ' \ + 'UTF-8 characters. To allow non-UTF-8 characters and ' \ + 'replace them with spaces, please set "coerce_to_utf8" ' \ + 'to true.' raise end end @@ -2174,8 +2168,8 @@ def construct_error_details_map(error) end error_details_map rescue JSON::ParserError => e - @log.warn 'Failed to extract log entry errors from the error details:' \ - " #{error.body}.", error: e + log.warn 'Failed to extract log entry errors from the error details:' \ + " #{error.body}.", error: e {} end @@ -2236,8 +2230,8 @@ def construct_error_details_map_grpc(gax_error) end error_details_map rescue JSON::ParserError => e - @log.warn 'Failed to extract log entry errors from the error details:' \ - " #{gax_error.details.inspect}.", error: e + log.warn 'Failed to extract log entry errors from the error details:' \ + " #{gax_error.details.inspect}.", error: e {} end @@ -2263,8 +2257,8 @@ def construct_k8s_resource_locally(local_resource_id) @k8s_cluster_location ||= fetch_gce_metadata( 'instance/attributes/cluster-location') rescue StandardError => e - @log.error 'Failed to retrieve k8s cluster name and location.', \ - error: e + log.error 'Failed to retrieve k8s cluster name and location.', \ + error: e end case resource_type when K8S_CONTAINER_CONSTANTS[:resource_type] @@ -2285,16 +2279,16 @@ def construct_k8s_resource_locally(local_resource_id) fallback_resource = COMPUTE_CONSTANTS[:resource_type] end unless @k8s_cluster_name && @k8s_cluster_location - @log.error "Failed to construct #{resource_type} resource locally." \ - ' Falling back to writing logs against' \ - " #{fallback_resource} resource.", error: e + log.error "Failed to construct #{resource_type} resource locally." \ + ' Falling back to writing logs against' \ + " #{fallback_resource} resource.", error: e return end constructed_resource = Google::Apis::LoggingV2::MonitoredResource.new( type: resource_type, labels: labels) - @log.debug("Constructed #{resource_type} resource locally: " \ - "#{constructed_resource.inspect}") + log.debug("Constructed #{resource_type} resource locally: " \ + "#{constructed_resource.inspect}") constructed_resource end From 2bd9641b9e4e4d54fdc61e2aaa7e974353554264 Mon Sep 17 00:00:00 2001 From: Kenji Okimoto Date: Thu, 22 Nov 2018 14:18:40 +0900 Subject: [PATCH 3/4] Remove needless rubocop annotation --- test/helper.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/helper.rb b/test/helper.rb index 52ad529a..b377fe4b 100644 --- a/test/helper.rb +++ b/test/helper.rb @@ -34,7 +34,7 @@ def method_missing(_method, *_args) end end # global $log variable is used by fluentd - $log = nulllogger # rubocop:disable Style/GlobalVars + $log = nulllogger end require 'fluent/plugin/out_google_cloud' From f9445ec0e4742cfc97e29988e7ca0eca1d118f9a Mon Sep 17 00:00:00 2001 From: okkez Date: Thu, 15 Aug 2019 14:05:15 +0900 Subject: [PATCH 4/4] Fix alignment reported by rubocop --- lib/fluent/plugin/out_google_cloud.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/fluent/plugin/out_google_cloud.rb b/lib/fluent/plugin/out_google_cloud.rb index fddd068f..efb4f22e 100644 --- a/lib/fluent/plugin/out_google_cloud.rb +++ b/lib/fluent/plugin/out_google_cloud.rb @@ -1264,7 +1264,7 @@ def determine_agent_level_monitored_resource_labels(type) rescue StandardError => e if [Platform::GCE, Platform::EC2].include?(@platform) log.error "Failed to set monitored resource labels for #{type}: ", - error: e + error: e end {} end