ZeroTierOne/node/Path.hpp

/*
 * ZeroTier One - Network Virtualization Everywhere
 * Copyright (C) 2011-2018  ZeroTier, Inc.  https://www.zerotier.com/
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 * --
 *
 * You can be released from the requirements of the license by purchasing
 * a commercial license. Buying such a license is mandatory as soon as you
 * develop commercial closed-source software that incorporates or links
 * directly against ZeroTier software without disclosing the source code
 * of your own application.
 */

#ifndef ZT_PATH_HPP
#define ZT_PATH_HPP

#include <stdint.h>
#include <string.h>
#include <stdlib.h>

#include <stdexcept>
#include <algorithm>

#include "Constants.hpp"
#include "InetAddress.hpp"
#include "SharedPtr.hpp"
#include "AtomicCounter.hpp"
#include "Utils.hpp"
#include "RingBuffer.hpp"

#include "../osdep/Phy.hpp"

/**
 * Maximum return value of preferenceRank()
 */
#define ZT_PATH_MAX_PREFERENCE_RANK ((ZT_INETADDRESS_MAX_SCOPE << 1) | 1)

namespace ZeroTier {

class RuntimeEnvironment;

/**
 * A path across the physical network
 */
class Path
{
	friend class SharedPtr<Path>;
	Phy<Path *> *_phy;

public:
	/**
	 * Efficient unique key for paths in a Hashtable
	 */
	class HashKey
	{
	public:
		HashKey() {}

		HashKey(const int64_t l,const InetAddress &r)
		{
			if (r.ss_family == AF_INET) {
				_k[0] = (uint64_t)reinterpret_cast<const struct sockaddr_in *>(&r)->sin_addr.s_addr;
				_k[1] = (uint64_t)reinterpret_cast<const struct sockaddr_in *>(&r)->sin_port;
				_k[2] = (uint64_t)l;
			} else if (r.ss_family == AF_INET6) {
				ZT_FAST_MEMCPY(_k,reinterpret_cast<const struct sockaddr_in6 *>(&r)->sin6_addr.s6_addr,16);
				_k[2] = ((uint64_t)reinterpret_cast<const struct sockaddr_in6 *>(&r)->sin6_port << 32) ^ (uint64_t)l;
			} else {
				ZT_FAST_MEMCPY(_k,&r,std::min(sizeof(_k),sizeof(InetAddress)));
				_k[2] += (uint64_t)l;
			}
		}

		inline unsigned long hashCode() const { return (unsigned long)(_k[0] + _k[1] + _k[2]); }

		inline bool operator==(const HashKey &k) const { return ( (_k[0] == k._k[0]) && (_k[1] == k._k[1]) && (_k[2] == k._k[2]) ); }
		inline bool operator!=(const HashKey &k) const { return (!(*this == k)); }

	private:
		uint64_t _k[3];
	};

	Path() :
		_lastOut(0),
		_lastIn(0),
		_lastTrustEstablishedPacketReceived(0),
		_lastPathQualityComputeTime(0),
		_localSocket(-1),
		_latency(0xffff),
		_addr(),
		_ipScope(InetAddress::IP_SCOPE_NONE),
		_currentPacketSampleCounter(0),
		_meanPacketErrorRatio(0.0),
		_meanLatency(0.0),
		_lastLatencyUpdate(0),
		_jitter(0.0),
		_lastPathQualitySampleTime(0),
		_lastComputedQuality(0.0),
		_lastPathQualityEstimate(0),
		_meanAge(0.0),
		_meanThroughput(0.0),
		_packetLossRatio(0)
	{
		memset(_ifname, 0, sizeof(_ifname));
		memset(_addrString, 0, sizeof(_addrString));
		_throughputSamples = new RingBuffer<uint64_t>(ZT_PATH_QUALITY_METRIC_WIN_SZ);
		_ageSamples = new RingBuffer<uint64_t>(ZT_PATH_QUALITY_METRIC_WIN_SZ);
		_latencySamples = new RingBuffer<uint32_t>(ZT_PATH_QUALITY_METRIC_WIN_SZ);
		_errSamples = new RingBuffer<float>(ZT_PATH_QUALITY_METRIC_WIN_SZ);
	}

	Path(const int64_t localSocket,const InetAddress &addr) :
		_lastOut(0),
		_lastIn(0),
		_lastTrustEstablishedPacketReceived(0),
		_lastPathQualityComputeTime(0),
		_localSocket(localSocket),
		_latency(0xffff),
		_addr(addr),
		_ipScope(addr.ipScope()),
		_currentPacketSampleCounter(0),
		_meanPacketErrorRatio(0.0),
		_meanLatency(0.0),
		_lastLatencyUpdate(0),
		_jitter(0.0),
		_lastPathQualitySampleTime(0),
		_lastComputedQuality(0.0),
		_lastPathQualityEstimate(0),
		_meanAge(0.0),
		_meanThroughput(0.0),
		_packetLossRatio(0)
	{
		memset(_ifname, 0, sizeof(_ifname));
		memset(_addrString, 0, sizeof(_addrString));
		_throughputSamples = new RingBuffer<uint64_t>(ZT_PATH_QUALITY_METRIC_WIN_SZ);
		_ageSamples = new RingBuffer<uint64_t>(ZT_PATH_QUALITY_METRIC_WIN_SZ);
		_latencySamples = new RingBuffer<uint32_t>(ZT_PATH_QUALITY_METRIC_WIN_SZ);
		_errSamples = new RingBuffer<float>(ZT_PATH_QUALITY_METRIC_WIN_SZ);
	}

	~Path()
	{
		delete _throughputSamples;
		delete _ageSamples;
		delete _latencySamples;
		delete _errSamples;

		_throughputSamples = NULL;
		_ageSamples = NULL;
		_latencySamples = NULL;
		_errSamples = NULL;
	}

	/**
	 * Called when a packet is received from this remote path, regardless of content
	 *
	 * @param t Time of receive
	 */
	inline void received(const uint64_t t) { _lastIn = t; }

	/**
	 * Set time last trusted packet was received (done in Peer::received())
	 */
	inline void trustedPacketReceived(const uint64_t t) { _lastTrustEstablishedPacketReceived = t; }

	/**
	 * Send a packet via this path (last out time is also updated)
	 *
	 * @param RR Runtime environment
	 * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
	 * @param data Packet data
	 * @param len Packet length
	 * @param now Current time
	 * @return True if transport reported success
	 */
	bool send(const RuntimeEnvironment *RR,void *tPtr,const void *data,unsigned int len,int64_t now);

	/**
	 * Manually update last sent time
	 *
	 * @param t Time of send
	 */
	inline void sent(const int64_t t) { _lastOut = t; }

	/**
	 * Update path latency with a new measurement
	 *
	 * @param l Measured latency
	 */
	inline void updateLatency(const unsigned int l, int64_t now)
	{
		unsigned int pl = _latency;
		if (pl < 0xffff) {
			_latency = (pl + l) / 2;
		}
		else {
			_latency = l;
		}
		_lastLatencyUpdate = now;
		_latencySamples->push(l);
	}

	/**
	 * @return Local socket as specified by external code
	 */
	inline int64_t localSocket() const { return _localSocket; }

	/**
	 * @return Physical address
	 */
	inline const InetAddress &address() const { return _addr; }

	/**
	 * @return IP scope -- faster shortcut for address().ipScope()
	 */
	inline InetAddress::IpScope ipScope() const { return _ipScope; }

	/**
	 * @return True if path has received a trust established packet (e.g. common network membership) in the past ZT_TRUST_EXPIRATION ms
	 */
	inline bool trustEstablished(const int64_t now) const { return ((now - _lastTrustEstablishedPacketReceived) < ZT_TRUST_EXPIRATION); }

	/**
	 * @return Preference rank, higher == better
	 */
	inline unsigned int preferenceRank() const
	{
		// This causes us to rank paths in order of IP scope rank (see InetAdddress.hpp) but
		// within each IP scope class to prefer IPv6 over IPv4.
		return ( ((unsigned int)_ipScope << 1) | (unsigned int)(_addr.ss_family == AF_INET6) );
	}

	/**
	 * Check whether this address is valid for a ZeroTier path
	 *
	 * This checks the address type and scope against address types and scopes
	 * that we currently support for ZeroTier communication.
	 *
	 * @param a Address to check
	 * @return True if address is good for ZeroTier path use
	 */
	static inline bool isAddressValidForPath(const InetAddress &a)
	{
		if ((a.ss_family == AF_INET)||(a.ss_family == AF_INET6)) {
			switch(a.ipScope()) {
				/* Note: we don't do link-local at the moment. Unfortunately these
				 * cause several issues. The first is that they usually require a
				 * device qualifier, which we don't handle yet and can't portably
				 * push in PUSH_DIRECT_PATHS. The second is that some OSes assign
				 * these very ephemerally or otherwise strangely. So we'll use
				 * private, pseudo-private, shared (e.g. carrier grade NAT), or
				 * global IP addresses. */
				case InetAddress::IP_SCOPE_PRIVATE:
				case InetAddress::IP_SCOPE_PSEUDOPRIVATE:
				case InetAddress::IP_SCOPE_SHARED:
				case InetAddress::IP_SCOPE_GLOBAL:
					if (a.ss_family == AF_INET6) {
						// TEMPORARY HACK: for now, we are going to blacklist he.net IPv6
						// tunnels due to very spotty performance and low MTU issues over
						// these IPv6 tunnel links.
						const uint8_t *ipd = reinterpret_cast<const uint8_t *>(reinterpret_cast<const struct sockaddr_in6 *>(&a)->sin6_addr.s6_addr);
						if ((ipd[0] == 0x20)&&(ipd[1] == 0x01)&&(ipd[2] == 0x04)&&(ipd[3] == 0x70))
							return false;
					}
					return true;
				default:
					return false;
			}
		}
		return false;
	}

	/**
	 * @return Latency or 0xffff if unknown
	 */
	inline unsigned int latency() const { return _latency; }

	/**
	 * @return Path quality -- lower is better
	 */
	inline long quality(const int64_t now) const
	{
		const int l = (long)_latency;
		const int age = (long)std::min((now - _lastIn),(int64_t)(ZT_PATH_HEARTBEAT_PERIOD * 10)); // set an upper sanity limit to avoid overflow
		return (((age < (ZT_PATH_HEARTBEAT_PERIOD + 5000)) ? l : (l + 0xffff + age)) * (long)((ZT_INETADDRESS_MAX_SCOPE - _ipScope) + 1));
	}

	/**
	 * @return An estimate of path quality -- higher is better.
	 */
	inline float computeQuality(const int64_t now)
	{
		float latency_contrib    = _meanLatency ? (float)1.0 / _meanLatency : 0;
		float jitter_contrib     = _jitter ? (float)1.0 / _jitter : 0;
		float throughput_contrib = _meanThroughput ? _meanThroughput / 1000000 : 0; // in Mbps
		float age_contrib        = _meanAge > 0 ? (float)sqrt(_meanAge) : 1;
		float error_contrib      = (float)1.0 - _meanPacketErrorRatio;
		float sum = (latency_contrib + jitter_contrib + throughput_contrib + error_contrib) / age_contrib;
		_lastComputedQuality = sum * (long)((_ipScope) + 1);
		return _lastComputedQuality;
	}

	/**
	 * Since quality estimates can become expensive we should cache the most recent result for traffic allocation
	 * algorithms which may need to reference this value multiple times through the course of their execution.
	 */
	inline float lastComputedQuality() {
		return _lastComputedQuality;
	}

	/**
	 * @return A pointer to a cached copy of the human-readable name of the interface this Path's localSocket is bound to
	 */
	inline char *getName() { return _ifname; }

	/**
	 * @return Estimated throughput in bps of this link
	 */
	inline uint64_t getThroughput() { return _phy->getThroughput((PhySocket *)((uintptr_t)_localSocket)); }

	/**
	 * @return Packet delay varience
	 */
	inline float jitter() { return _jitter; }

	/**
	 * @return Previously-computed mean latency
	 */
	inline float meanLatency() { return _meanLatency; }

	/**
	 * @return Packet loss rate
	 */
	inline float packetLossRatio() { return _packetLossRatio; }

	/**
	 * @return Mean packet error ratio
	 */
	inline float meanPacketErrorRatio() { return _meanPacketErrorRatio; }

	/**
	 * @return Current packet error ratio (possibly incomplete sample set)
	 */
	inline float currentPacketErrorRatio() {
		int errorsPerSample = 0;
		for (int i=0; i<_currentPacketSampleCounter; i++) {
			if (_packetValidity[i] == false) {
				errorsPerSample++;
			}
		}
		return (float)errorsPerSample / (float)ZT_PATH_ERROR_SAMPLE_WIN_SZ;
	}

	/**
	 * @return Whether the Path's local socket is in a CLOSED state
	 */
	inline bool isClosed() { return _phy->isClosed((PhySocket *)((uintptr_t)_localSocket)); }

	/**
	 * @return The state of a Path's local socket
	 */
	inline int getState() { return _phy->getState((PhySocket *)((uintptr_t)_localSocket)); }

	/**
	 * @return Whether this socket may have been erased by the virtual physical link layer
	 */
	inline bool isValidState() { return _phy->isValidState((PhySocket *)((uintptr_t)_localSocket)); }

	/**
	 * @return Whether the path quality monitors have collected enough data to provide a quality value
	 * TODO: expand this
	 */
	inline bool monitorsReady() {
		return _latencySamples->count() && _ageSamples->count() && _throughputSamples->count();
	}

	/**
	 * @return A pointer to a cached copy of the address string for this Path (For debugging only)
	 */
	inline char *getAddressString() { return _addrString; }

	/**
	 * Handle path sampling, computation of quality estimates, and other periodic tasks
	 * @param now Current time
	 */
	inline void measureLink(int64_t now) {
		// Sample path properties and store them in a continuously-revolving buffer
		if (now - _lastPathQualitySampleTime > ZT_PATH_QUALITY_SAMPLE_INTERVAL) {
			_lastPathQualitySampleTime = now;
			_throughputSamples->push(getThroughput()); // Thoughtput in bits/s
			_ageSamples->push(now - _lastIn); // Age (time since last received packet)
			if (now - _lastLatencyUpdate > ZT_PATH_LATENCY_SAMPLE_INTERVAL) {
				_lastLatencyUpdate = now;
				// Record 0 bp/s. Since we're using this to detect possible packet loss
				updateLatency(0, now);
			}
		}
		// Compute statistical values for use in link quality estimates
		if (now - _lastPathQualityComputeTime > ZT_PATH_QUALITY_COMPUTE_INTERVAL) {
			_lastPathQualityComputeTime = now;
			// Cache Path address string
			address().toString(_addrString);
			_phy->getIfName((PhySocket *)((uintptr_t)_localSocket), _ifname, ZT_PATH_INTERFACE_NAME_SZ); // Cache Interface name
			// Derived values
			if (_throughputSamples->count()) {
				_packetLossRatio = (float)_throughputSamples->zeroCount() / (float)_throughputSamples->count();
			}
			_meanThroughput = _throughputSamples->mean();
			_meanAge = _ageSamples->mean();
			_meanLatency = _latencySamples->mean();
			// Jitter
			// SEE: RFC 3393, RFC 4689
			_jitter = _latencySamples->stddev();
			_meanPacketErrorRatio = _errSamples->mean(); // Packet Error Ratio (PER)
		}
		// Periodically compute a path quality estimate
		if (now - _lastPathQualityEstimate > ZT_PATH_QUALITY_ESTIMATE_INTERVAL) {
			computeQuality(now);
		}
	}

	/**
	 * @param buf Buffer to store resultant string
	 * @return Description of path, in ASCII string format
	 */
	inline char *toString(char *buf) {
		sprintf(buf,"%6s, q=%8.3f, %5.3f Mb/s, j=%8.2f, ml=%8.2f, meanAge=%8.2f, addr=%45s",
			getName(),
			lastComputedQuality(),
			(float)meanThroughput() / (float)1000000,
			jitter(),
			meanLatency(),
			meanAge(),
			getAddressString());
		return buf;
	}

	/**
	 * Record whether a packet is considered invalid by MAC/compression/cipher checks. This
	 * could be an indication of a bit error. This function will keep a running counter of
	 * up to a given window size and with each counter overflow it will compute a mean error rate
	 * and store that in a continuously shifting sample window.
	 *
	 * @param isValid Whether the packet in question is considered invalid
	 */
	inline void recordPacket(bool isValid) {
		if (_currentPacketSampleCounter < ZT_PATH_ERROR_SAMPLE_WIN_SZ) {
			_packetValidity[_currentPacketSampleCounter] = isValid;
			_currentPacketSampleCounter++;
		}
		else {
			// Sample array is full, compute an mean and stick it in the ring buffer for trend analysis
			_errSamples->push(currentPacketErrorRatio());
			_currentPacketSampleCounter=0;
		}
	}

	/**
	 * @return The mean age (in ms) of this link
	 */
	inline float meanAge() { return _meanAge; }

	/**
	 * @return The mean throughput (in bits/s) of this link
	 */
	inline float meanThroughput() { return _meanThroughput; }

	/**
	 * @return True if this path is alive (receiving heartbeats)
	 */
	inline bool alive(const int64_t now) const { return ((now - _lastIn) < (ZT_PATH_HEARTBEAT_PERIOD + 5000)); }

	/**
	 * @return True if this path hasn't received a packet in a "significant" amount of time
	 */
	inline bool stale(const int64_t now) const { return ((now - _lastIn) > ZT_LINK_SPEED_TEST_INTERVAL * 10); }

	/**
	 * @return True if this path needs a heartbeat
	 */
	inline bool needsHeartbeat(const int64_t now) const { return ((now - _lastOut) >= ZT_PATH_HEARTBEAT_PERIOD); }

	/**
	 * @return Last time we sent something
	 */
	inline int64_t lastOut() const { return _lastOut; }

	/**
	 * @return Last time we received anything
	 */
	inline int64_t lastIn() const { return _lastIn; }

	/**
	 * @return Time last trust-established packet was received
	 */
	inline int64_t lastTrustEstablishedPacketReceived() const { return _lastTrustEstablishedPacketReceived; }

private:
	volatile int64_t _lastOut;
	volatile int64_t _lastIn;
	volatile int64_t _lastTrustEstablishedPacketReceived;
	volatile int64_t _lastPathQualityComputeTime;
	int64_t _localSocket;
	volatile unsigned int _latency;
	InetAddress _addr;
	InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often
	AtomicCounter __refCount;

	// Packet Error Ratio (PER)
	int _packetValidity[ZT_PATH_ERROR_SAMPLE_WIN_SZ];
	int _currentPacketSampleCounter;
	volatile float _meanPacketErrorRatio;

	// Latency and Jitter
	volatile float _meanLatency;
	int64_t _lastLatencyUpdate;
	volatile float _jitter;

	int64_t _lastPathQualitySampleTime;
	float _lastComputedQuality;
	int64_t _lastPathQualityEstimate;
	float _meanAge;
	float _meanThroughput;

	// Circular buffers used to efficiently store large time series
	RingBuffer<uint64_t> *_throughputSamples;
	RingBuffer<uint32_t> *_latencySamples;
	RingBuffer<uint64_t> *_ageSamples;
	RingBuffer<float> *_errSamples;

	float _packetLossRatio;

	char _ifname[ZT_PATH_INTERFACE_NAME_SZ];
	char _addrString[256];
};

} // namespace ZeroTier

#endif