network_engine.h
/*
* Copyright (C) 2014-2018 Savoir-faire Linux Inc.
* Author(s) : Adrien Béraud <adrien.beraud@savoirfairelinux.com>
* Simon Désaulniers <simon.desaulniers@savoirfairelinux.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
#pragma once
#include "node_cache.h"
#include "value.h"
#include "infohash.h"
#include "node.h"
#include "scheduler.h"
#include "utils.h"
#include "rng.h"
#include "rate_limiter.h"
#include <vector>
#include <string>
#include <functional>
#include <algorithm>
#include <memory>
#include <queue>
namespace dht {
namespace net {
struct Request;
struct Socket;
struct TransId;
#ifndef MSG_CONFIRM
#define MSG_CONFIRM 0
#endif
class DhtProtocolException : public DhtException {
public:
// sent to another peer (http-like).
static const constexpr uint16_t NON_AUTHORITATIVE_INFORMATION {203}; /* incomplete request packet. */
static const constexpr uint16_t UNAUTHORIZED {401}; /* wrong tokens. */
static const constexpr uint16_t NOT_FOUND {404}; /* storage not found */
// for internal use (custom).
static const constexpr uint16_t INVALID_TID_SIZE {421}; /* id was truncated. */
static const constexpr uint16_t UNKNOWN_TID {422}; /* unknown tid */
static const constexpr uint16_t WRONG_NODE_INFO_BUF_LEN {423}; /* node info length is wrong */
static const std::string GET_NO_INFOHASH; /* received "get" request with no infohash */
static const std::string LISTEN_NO_INFOHASH; /* got "listen" request without infohash */
static const std::string LISTEN_WRONG_TOKEN; /* wrong token in "listen" request */
static const std::string PUT_NO_INFOHASH; /* no infohash in "put" request */
static const std::string PUT_WRONG_TOKEN; /* got "put" request with wrong token */
static const std::string STORAGE_NOT_FOUND; /* got access request for an unknown storage */
static const std::string PUT_INVALID_ID; /* invalid id in "put" request */
DhtProtocolException(uint16_t code, const std::string& msg="", InfoHash failing_node_id={})
: DhtException(msg), msg(msg), code(code), failing_node_id(failing_node_id) {}
std::string getMsg() const { return msg; }
uint16_t getCode() const { return code; }
const InfoHash getNodeId() const { return failing_node_id; }
private:
std::string msg;
uint16_t code;
const InfoHash failing_node_id;
};
struct ParsedMessage;
struct RequestAnswer {
Blob ntoken {};
Value::Id vid {};
std::vector<Sp<Value>> values {};
std::vector<Value::Id> refreshed_values {};
std::vector<Value::Id> expired_values {};
std::vector<Sp<FieldValueIndex>> fields {};
std::vector<Sp<Node>> nodes4 {};
std::vector<Sp<Node>> nodes6 {};
RequestAnswer() {}
RequestAnswer(ParsedMessage&& msg);
};
class NetworkEngine final
{
private:
std::function<void(Sp<Request>, DhtProtocolException)> onError;
std::function<void(const Sp<Node>&, int)> onNewNode;
std::function<void(const InfoHash&, const SockAddr&)> onReportedAddr;
std::function<RequestAnswer(Sp<Node>)> onPing {};
std::function<RequestAnswer(Sp<Node>, const InfoHash&, want_t)> onFindNode {};
std::function<RequestAnswer(Sp<Node>, const InfoHash&, want_t, const Query&)> onGetValues {};
std::function<RequestAnswer(Sp<Node>,
const InfoHash&,
const Blob&,
Tid,
const Query&)> onListen {};
std::function<RequestAnswer(Sp<Node>,
const InfoHash&,
const Blob&,
const std::vector<Sp<Value>>&,
const time_point&)> onAnnounce {};
std::function<RequestAnswer(Sp<Node>,
const InfoHash&,
const Blob&,
const Value::Id&)> onRefresh {};
public:
using RequestCb = std::function<void(const Request&, RequestAnswer&&)>;
using RequestExpiredCb = std::function<void(const Request&, bool)>;
NetworkEngine(Logger& log, Scheduler& scheduler, const int& s = -1, const int& s6 = -1);
NetworkEngine(InfoHash& myid, NetId net, const int& s, const int& s6, Logger& log, Scheduler& scheduler,
decltype(NetworkEngine::onError) onError,
decltype(NetworkEngine::onNewNode) onNewNode,
decltype(NetworkEngine::onReportedAddr) onReportedAddr,
decltype(NetworkEngine::onPing) onPing,
decltype(NetworkEngine::onFindNode) onFindNode,
decltype(NetworkEngine::onGetValues) onGetValues,
decltype(NetworkEngine::onListen) onListen,
decltype(NetworkEngine::onAnnounce) onAnnounce,
decltype(NetworkEngine::onRefresh) onRefresh);
virtual ~NetworkEngine();
void clear();
void tellListener(Sp<Node> n, Tid socket_id, const InfoHash& hash, want_t want, const Blob& ntoken,
std::vector<Sp<Node>>&& nodes, std::vector<Sp<Node>>&& nodes6,
std::vector<Sp<Value>>&& values, const Query& q);
void tellListenerRefreshed(Sp<Node> n, Tid socket_id, const InfoHash& hash, const Blob& ntoken, const std::vector<Value::Id>& values);
void tellListenerExpired(Sp<Node> n, Tid socket_id, const InfoHash& hash, const Blob& ntoken, const std::vector<Value::Id>& values);
bool isRunning(sa_family_t af) const;
inline want_t want () const { return dht_socket >= 0 && dht_socket6 >= 0 ? (WANT4 | WANT6) : -1; }
void connectivityChanged(sa_family_t);
/**************
* Requests *
**************/
Sp<Request>
sendPing(Sp<Node> n, RequestCb&& on_done, RequestExpiredCb&& on_expired);
Sp<Request>
sendPing(const sockaddr* sa, socklen_t salen, RequestCb&& on_done, RequestExpiredCb&& on_expired) {
return sendPing(std::make_shared<Node>(zeroes, sa, salen),
std::forward<RequestCb>(on_done),
std::forward<RequestExpiredCb>(on_expired));
}
Sp<Request> sendFindNode(Sp<Node> n,
const InfoHash& hash,
want_t want,
RequestCb&& on_done,
RequestExpiredCb&& on_expired);
Sp<Request> sendGetValues(Sp<Node> n,
const InfoHash& hash,
const Query& query,
want_t want,
RequestCb&& on_done,
RequestExpiredCb&& on_expired);
Sp<Request> sendListen(Sp<Node> n,
const InfoHash& hash,
const Query& query,
const Blob& token,
Sp<Request> previous,
RequestCb&& on_done,
RequestExpiredCb&& on_expired,
SocketCb&& socket_cb);
Sp<Request> sendAnnounceValue(Sp<Node> n,
const InfoHash& hash,
const Sp<Value>& v,
time_point created,
const Blob& token,
RequestCb&& on_done,
RequestExpiredCb&& on_expired);
Sp<Request> sendRefreshValue(Sp<Node> n,
const InfoHash& hash,
const Value::Id& vid,
const Blob& token,
RequestCb&& on_done,
RequestExpiredCb&& on_expired);
void processMessage(const uint8_t *buf, size_t buflen, const SockAddr& addr);
Sp<Node> insertNode(const InfoHash& myid, const SockAddr& addr) {
auto n = cache.getNode(myid, addr, scheduler.time(), 0);
onNewNode(n, 0);
return n;
}
std::vector<unsigned> getNodeMessageStats(bool in) {
auto& st = in ? in_stats : out_stats;
std::vector<unsigned> stats {st.ping, st.find, st.get, st.listen, st.put};
st = {};
return stats;
}
void blacklistNode(const Sp<Node>& n);
std::vector<Sp<Node>> getCachedNodes(const InfoHash& id, sa_family_t sa_f, size_t count) {
return cache.getCachedNodes(id, sa_f, count);
}
private:
struct PartialMessage;
/***************
* Constants *
***************/
static constexpr size_t MAX_REQUESTS_PER_SEC {1600};
/* the length of a node info buffer in ipv4 format */
static const constexpr size_t NODE4_INFO_BUF_LEN {HASH_LEN + sizeof(in_addr) + sizeof(in_port_t)};
/* the length of a node info buffer in ipv6 format */
static const constexpr size_t NODE6_INFO_BUF_LEN {HASH_LEN + sizeof(in6_addr) + sizeof(in_port_t)};
/* after a UDP reply, the period during which we tell the link layer about it */
static constexpr std::chrono::seconds UDP_REPLY_TIME {15};
/* Max. time to receive a full fragmented packet */
static constexpr std::chrono::seconds RX_MAX_PACKET_TIME {10};
/* Max. time between packet fragments */
static constexpr std::chrono::seconds RX_TIMEOUT {3};
/* The maximum number of nodes that we snub. There is probably little
reason to increase this value. */
static constexpr unsigned BLACKLISTED_MAX {10};
static constexpr size_t MTU {1280};
static constexpr size_t MAX_PACKET_VALUE_SIZE {600};
static const std::string my_v;
void process(std::unique_ptr<ParsedMessage>&&, const SockAddr& from);
bool rateLimit(const SockAddr& addr);
static bool isMartian(const SockAddr& addr);
bool isNodeBlacklisted(const SockAddr& addr) const;
void requestStep(Sp<Request> req);
void sendRequest(const Sp<Request>& request);
struct MessageStats {
unsigned ping {0};
unsigned find {0};
unsigned get {0};
unsigned put {0};
unsigned listen {0};
unsigned refresh {0};
};
// basic wrapper for socket sendto function
int send(const char *buf, size_t len, int flags, const SockAddr& addr);
void sendValueParts(const TransId& tid, const std::vector<Blob>& svals, const SockAddr& addr);
std::vector<Blob> packValueHeader(msgpack::sbuffer&, const std::vector<Sp<Value>>&);
void maintainRxBuffer(Tid tid);
/*************
* Answers *
*************/
/* answer to a ping request */
void sendPong(const SockAddr& addr, Tid tid);
/* answer to findnodes/getvalues request */
void sendNodesValues(const SockAddr& addr,
Tid tid,
const Blob& nodes,
const Blob& nodes6,
const std::vector<Sp<Value>>& st,
const Query& query,
const Blob& token);
Blob bufferNodes(sa_family_t af, const InfoHash& id, std::vector<Sp<Node>>& nodes);
std::pair<Blob, Blob> bufferNodes(sa_family_t af,
const InfoHash& id,
want_t want,
std::vector<Sp<Node>>& nodes,
std::vector<Sp<Node>>& nodes6);
/* answer to a listen request */
void sendListenConfirmation(const SockAddr& addr, Tid tid);
/* answer to put request */
void sendValueAnnounced(const SockAddr& addr, Tid, Value::Id);
/* answer in case of error */
void sendError(const SockAddr& addr,
Tid tid,
uint16_t code,
const std::string& message,
bool include_id=false);
void deserializeNodes(ParsedMessage& msg, const SockAddr& from);
/* DHT info */
const InfoHash& myid;
const NetId network {0};
const int& dht_socket;
const int& dht_socket6;
const Logger& DHT_LOG;
NodeCache cache {};
// global limiting should be triggered by at least 8 different IPs
using IpLimiter = RateLimiter<MAX_REQUESTS_PER_SEC/8>;
using IpLimiterMap = std::map<SockAddr, IpLimiter, SockAddr::ipCmp>;
IpLimiterMap address_rate_limiter {};
RateLimiter<MAX_REQUESTS_PER_SEC> rate_limiter {};
size_t limiter_maintenance {0};
// requests handling
std::map<Tid, Sp<Request>> requests {};
std::map<Tid, PartialMessage> partial_messages;
MessageStats in_stats {}, out_stats {};
std::set<SockAddr> blacklist {};
Scheduler& scheduler;
};
} /* namespace net */
} /* namespace dht */
dht::DhtException
Definition: utils.h:63
dht::Hash< HASH_LEN >
dht::Scheduler
Job scheduler.
Definition: scheduler.h:37
dht::Scheduler::time
const time_point & time() const
Definition: scheduler.h:116
dht::SockAddr
Definition: sockaddr.h:55
dht::net::DhtProtocolException
Definition: network_engine.h:49
dht::net::NetworkEngine
An abstraction of communication protocol on the network.
Definition: network_engine.h:118
dht::net::NetworkEngine::sendListen
Sp< Request > sendListen(Sp< Node > n, const InfoHash &hash, const Query &query, const Blob &token, Sp< Request > previous, RequestCb &&on_done, RequestExpiredCb &&on_expired, SocketCb &&socket_cb)
dht::net::NetworkEngine::sendPing
Sp< Request > sendPing(const sockaddr *sa, socklen_t salen, RequestCb &&on_done, RequestExpiredCb &&on_expired)
Definition: network_engine.h:274
dht::net::NetworkEngine::tellListener
void tellListener(Sp< Node > n, Tid socket_id, const InfoHash &hash, want_t want, const Blob &ntoken, std::vector< Sp< Node > > &&nodes, std::vector< Sp< Node > > &&nodes6, std::vector< Sp< Value > > &&values, const Query &q)
dht::net::NetworkEngine::processMessage
void processMessage(const uint8_t *buf, size_t buflen, const SockAddr &addr)
dht::net::NetworkEngine::sendAnnounceValue
Sp< Request > sendAnnounceValue(Sp< Node > n, const InfoHash &hash, const Sp< Value > &v, time_point created, const Blob &token, RequestCb &&on_done, RequestExpiredCb &&on_expired)
dht::net::NetworkEngine::sendFindNode
Sp< Request > sendFindNode(Sp< Node > n, const InfoHash &hash, want_t want, RequestCb &&on_done, RequestExpiredCb &&on_expired)
dht::net::NetworkEngine::sendGetValues
Sp< Request > sendGetValues(Sp< Node > n, const InfoHash &hash, const Query &query, want_t want, RequestCb &&on_done, RequestExpiredCb &&on_expired)
dht::net::NetworkEngine::sendRefreshValue
Sp< Request > sendRefreshValue(Sp< Node > n, const InfoHash &hash, const Value::Id &vid, const Blob &token, RequestCb &&on_done, RequestExpiredCb &&on_expired)
dht::net::NetworkEngine::sendPing
Sp< Request > sendPing(Sp< Node > n, RequestCb &&on_done, RequestExpiredCb &&on_expired)
dht
Definition: callbacks.h:34
dht::Blob
std::vector< uint8_t > Blob
Definition: utils.h:114
dht::Logger
Definition: log_enable.h:87
dht::Query
Describes a query destined to another peer.
Definition: value.h:864
dht::net::RequestAnswer
Definition: network_engine.h:86