Merge pull request #16 from lightstep/add-lightstepreceiver

Initial import of lightstepreceiver

Author: nslaughter

collector/components/lightstepreceiver/DESIGN.md

@@ -0,0 +1,43 @@
+# Design
+
+## Summary
+
+This receiver exposes *very* basic functionality, with only http/protobuf support
+(initially). Details are:
+
+* `ReportRequest` is the protobuf we send/receive, with `ReportRequest.Report`
+  being similar to `Resource` (e.g. `Resource` has attributes in its `Tags` attribute).
+* Legacy tracers send the service name as `lightstep.component_name` in
+  `ReportRequest.Report.Tags`, and we derive the actual OTel `service.name`
+  from it, falling back to `unknown_service`.
+* We do a **raw** ingestion/conversion, meaning we don't do any semconv mapping,
+ other than deriving `service.name` from `lightstep.component_name`. See
+ TODO below.
+* Legacy tracers send 64 bits TraceIds, which we convert to 128 bytes OTel ids.
+* Clock correction: Some legacy tracers (Java) perform clock correction, sending
+ along a timeoffset to be applied, and expecting back Receive/Transmit
+ timestamps from the microsatellites/collector:
+ - `ReportRequest`: `TimestampOffsetMicros` includes an offset that MUST
+   be applied to all timestamps being reported. This value is zero if
+   no clock correction is required.
+ - `ReportResponse`: This receiver sends two timestamps, `Receive` and
+   `Transmit`, with the times at which the latest request was received
+   and later answered with a response, in order to help the tracers
+   adjust their offsets.
+
+## TODO
+
+* Use `receiverhelper` mechanism for standard component observability signals.
+* Legacy tracers send payloads using the `application/octet-stream` content type and using the
+  `/api/v2/reports` path. We don't check for it but worth verifying this (at least the
+  content-type).
+* Top level `ReporterId` is not being used at this moment.
+* `Baggage` is being sent as part of Lightstep's `SpanContext`, but it is not exported in any way at this point.
+* Find all special Tags (e.g. "lightstep.*") and think which ones we should map.
+* Implement gRPC support.
+* Implement Thrift support.
+* Consider mapping semantic conventions:
+  - Values that can be consumed within the processor, e.g. detect event names from Logs, detect `StatusKind` from error tags.
+    - Consider using the OpenTracing compatibilty section in the Specification, which states how to process errors and multiple parents.
+  - Values that affect the entire OT ecosystem. Probably can be offered as a separate processor instead.
+  - Lightstep-specific tags (attributes) that _may_ need to be mapped to become useful for OTel processors.

collector/components/lightstepreceiver/big_endian_converter.go

@@ -0,0 +1,27 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+// Partially copied from github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/idutils
+
+package lightstepreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/lightstepreceiver"
+
+import (
+	"encoding/binary"
+
+	"go.opentelemetry.io/collector/pdata/pcommon"
+)
+
+// UInt64ToTraceID converts the pair of uint64 representation of a TraceID to pcommon.TraceID.
+func UInt64ToTraceID(high, low uint64) pcommon.TraceID {
+	traceID := [16]byte{}
+	binary.BigEndian.PutUint64(traceID[:8], high)
+	binary.BigEndian.PutUint64(traceID[8:], low)
+	return traceID
+}
+
+// UInt64ToSpanID converts the uint64 representation of a SpanID to pcommon.SpanID.
+func UInt64ToSpanID(id uint64) pcommon.SpanID {
+	spanID := [8]byte{}
+	binary.BigEndian.PutUint64(spanID[:], id)
+	return pcommon.SpanID(spanID)
+}

collector/components/lightstepreceiver/config.go

@@ -0,0 +1,64 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package lightstepreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/lightstepreceiver"
+
+import (
+	"errors"
+	"fmt"
+
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/config/confighttp"
+	"go.opentelemetry.io/collector/confmap"
+)
+
+const (
+	protocolsFieldName = "protocols"
+	protoHTTP          = "http"
+)
+
+type HTTPConfig struct {
+	*confighttp.ServerConfig `mapstructure:",squash"`
+}
+
+// Protocols is the configuration for the supported protocols.
+type Protocols struct {
+	HTTP *HTTPConfig `mapstructure:"http"`
+}
+
+// Config defines configuration for the Lightstep receiver.
+type Config struct {
+	// Protocols is the configuration for the supported protocols, currently HTTP.
+	Protocols `mapstructure:"protocols"`
+}
+
+var _ component.Config = (*Config)(nil)
+
+// Validate checks the receiver configuration is valid
+func (cfg *Config) Validate() error {
+	if cfg.HTTP == nil {
+		return errors.New("must specify at least one protocol when using the Lightstep receiver")
+	}
+	return nil
+}
+
+// Unmarshal a confmap.Conf into the config struct.
+func (cfg *Config) Unmarshal(componentParser *confmap.Conf) error {
+	if componentParser == nil {
+		return fmt.Errorf("nil config for lightstepreceiver")
+	}
+
+	err := componentParser.Unmarshal(cfg)
+	if err != nil {
+		return err
+	}
+	protocols, err := componentParser.Sub(protocolsFieldName)
+	if err != nil {
+		return err
+	}
+
+	if !protocols.IsSet(protoHTTP) {
+		cfg.HTTP = nil
+	}
+	return nil
+}

collector/components/lightstepreceiver/config_test.go

@@ -0,0 +1,121 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package lightstepreceiver
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/config/confighttp"
+	"go.opentelemetry.io/collector/config/configtls"
+	"go.opentelemetry.io/collector/confmap"
+	"go.opentelemetry.io/collector/confmap/confmaptest"
+)
+
+func TestUnmarshalDefaultConfig(t *testing.T) {
+	cm, err := confmaptest.LoadConf(filepath.Join("testdata", "default.yaml"))
+	require.NoError(t, err)
+	factory := NewFactory()
+	cfg := factory.CreateDefaultConfig()
+	assert.NoError(t, component.UnmarshalConfig(cm, cfg))
+	assert.Equal(t, factory.CreateDefaultConfig(), cfg)
+}
+
+func TestUnmarshalConfigOnlyHTTP(t *testing.T) {
+	cm, err := confmaptest.LoadConf(filepath.Join("testdata", "only_http.yaml"))
+	require.NoError(t, err)
+	factory := NewFactory()
+	cfg := factory.CreateDefaultConfig()
+	assert.NoError(t, component.UnmarshalConfig(cm, cfg))
+
+	defaultOnlyHTTP := factory.CreateDefaultConfig().(*Config)
+	assert.Equal(t, defaultOnlyHTTP, cfg)
+}
+
+func TestUnmarshalConfigOnlyHTTPNull(t *testing.T) {
+	cm, err := confmaptest.LoadConf(filepath.Join("testdata", "only_http_null.yaml"))
+	require.NoError(t, err)
+	factory := NewFactory()
+	cfg := factory.CreateDefaultConfig()
+	assert.NoError(t, component.UnmarshalConfig(cm, cfg))
+
+	defaultOnlyHTTP := factory.CreateDefaultConfig().(*Config)
+	assert.Equal(t, defaultOnlyHTTP, cfg)
+}
+
+func TestUnmarshalConfigOnlyHTTPEmptyMap(t *testing.T) {
+	cm, err := confmaptest.LoadConf(filepath.Join("testdata", "only_http_empty_map.yaml"))
+	require.NoError(t, err)
+	factory := NewFactory()
+	cfg := factory.CreateDefaultConfig()
+	assert.NoError(t, component.UnmarshalConfig(cm, cfg))
+
+	defaultOnlyHTTP := factory.CreateDefaultConfig().(*Config)
+	assert.Equal(t, defaultOnlyHTTP, cfg)
+}
+
+func TestUnmarshalConfig(t *testing.T) {
+	cm, err := confmaptest.LoadConf(filepath.Join("testdata", "config.yaml"))
+	require.NoError(t, err)
+	factory := NewFactory()
+	cfg := factory.CreateDefaultConfig()
+	assert.NoError(t, component.UnmarshalConfig(cm, cfg))
+	assert.Equal(t,
+		&Config{
+			Protocols: Protocols{
+				HTTP: &HTTPConfig{
+					ServerConfig: &confighttp.ServerConfig{
+						Endpoint: "0.0.0.0:443",
+						TLSSetting: &configtls.ServerConfig{
+							Config: configtls.Config{
+								CertFile: "test.crt",
+								KeyFile:  "test.key",
+							},
+						},
+						CORS: &confighttp.CORSConfig{
+							AllowedOrigins: []string{"https://*.test.com", "https://test.com"},
+							MaxAge:         7200,
+						},
+					},
+				},
+			},
+		}, cfg)
+
+}
+
+func TestUnmarshalConfigTypoDefaultProtocol(t *testing.T) {
+	cm, err := confmaptest.LoadConf(filepath.Join("testdata", "typo_default_proto_config.yaml"))
+	require.NoError(t, err)
+	factory := NewFactory()
+	cfg := factory.CreateDefaultConfig()
+	assert.EqualError(t, component.UnmarshalConfig(cm, cfg), "1 error(s) decoding:\n\n* 'protocols' has invalid keys: htttp")
+}
+
+func TestUnmarshalConfigInvalidProtocol(t *testing.T) {
+	cm, err := confmaptest.LoadConf(filepath.Join("testdata", "bad_proto_config.yaml"))
+	require.NoError(t, err)
+	factory := NewFactory()
+	cfg := factory.CreateDefaultConfig()
+	assert.EqualError(t, component.UnmarshalConfig(cm, cfg), "1 error(s) decoding:\n\n* 'protocols' has invalid keys: thrift")
+}
+
+func TestUnmarshalConfigEmptyProtocols(t *testing.T) {
+	cm, err := confmaptest.LoadConf(filepath.Join("testdata", "bad_no_proto_config.yaml"))
+	require.NoError(t, err)
+	factory := NewFactory()
+	cfg := factory.CreateDefaultConfig()
+	assert.NoError(t, component.UnmarshalConfig(cm, cfg))
+	assert.EqualError(t, component.ValidateConfig(cfg), "must specify at least one protocol when using the Lightstep receiver")
+}
+
+func TestUnmarshalConfigEmpty(t *testing.T) {
+	factory := NewFactory()
+	cfg := factory.CreateDefaultConfig()
+	assert.NoError(t, component.UnmarshalConfig(confmap.New(), cfg))
+	assert.EqualError(t, component.ValidateConfig(cfg), "must specify at least one protocol when using the Lightstep receiver")
+}

collector/components/lightstepreceiver/factory.go

@@ -0,0 +1,55 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package lightstepreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/lightstepreceiver"
+
+import (
+	"context"
+
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/config/confighttp"
+	"go.opentelemetry.io/collector/consumer"
+	"go.opentelemetry.io/collector/receiver"
+
+	"github.com/lightstep/sn-collector/collector/lightstepreceiver/internal/metadata"
+)
+
+// This file implements factory bits for the Lightstep receiver.
+
+const (
+	// TODO: Define a new port for us to use.
+	defaultBindEndpoint = "0.0.0.0:443"
+)
+
+// NewFactory creates a new Lightstep receiver factory
+func NewFactory() receiver.Factory {
+	return receiver.NewFactory(
+		metadata.Type,
+		createDefaultConfig,
+		receiver.WithTraces(createTracesReceiver, metadata.TracesStability),
+	)
+}
+
+// createDefaultConfig creates the default configuration for Lightstep receiver.
+func createDefaultConfig() component.Config {
+	return &Config{
+		Protocols: Protocols{
+			HTTP: &HTTPConfig{
+				ServerConfig: &confighttp.ServerConfig{
+					Endpoint: defaultBindEndpoint,
+				},
+			},
+		},
+	}
+}
+
+// createTracesReceiver creates a trace receiver based on provided config.
+func createTracesReceiver(
+	_ context.Context,
+	set receiver.CreateSettings,
+	cfg component.Config,
+	consumer consumer.Traces,
+) (receiver.Traces, error) {
+	rCfg := cfg.(*Config)
+	return newReceiver(rCfg, consumer, set)
+}

collector/components/lightstepreceiver/factory_test.go

@@ -0,0 +1,41 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package lightstepreceiver
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"go.opentelemetry.io/collector/component/componenttest"
+	"go.opentelemetry.io/collector/consumer/consumertest"
+	"go.opentelemetry.io/collector/receiver/receivertest"
+)
+
+func TestCreateDefaultConfig(t *testing.T) {
+	cfg := createDefaultConfig()
+	assert.NotNil(t, cfg, "failed to create default config")
+	assert.NoError(t, componenttest.CheckConfigStruct(cfg))
+}
+
+func TestCreateReceiver(t *testing.T) {
+	cfg := createDefaultConfig()
+
+	tReceiver, err := createTracesReceiver(
+		context.Background(),
+		receivertest.NewNopCreateSettings(),
+		cfg,
+		consumertest.NewNop())
+	assert.NoError(t, err, "receiver creation failed")
+	assert.NotNil(t, tReceiver, "receiver creation failed")
+
+	tReceiver, err = createTracesReceiver(
+		context.Background(),
+		receivertest.NewNopCreateSettings(),
+		cfg,
+		consumertest.NewNop())
+	assert.NoError(t, err, "receiver creation failed")
+	assert.NotNil(t, tReceiver, "receiver creation failed")
+}