全部文档
当前文档
最近更新时间：2024-03-26 14:09:23
四层负载均衡，如果您的后端为云主机时，在后端KEC上获取的源IP即为客户端 IP，可以通过后端服务器中的业务日志查看；
如果您的后端为裸金属，则需要按照本文操作步骤安装TTM模块来获取客户端源IP。
七层负载均衡，无论您的后端是KEC还是EPC，SLB 不再透传源 IP，您可以通过 X-Forwarded-For字段来直接获取客户端 IP。
本文主要介绍针对四层负载均衡，后端为裸金属时，如何获取客户端源IP。用户在主机（RS）侧加载内核模块 kgwttm.ko 后，对于 TCP 连接：可以通过 socket 函数簇函数获取客户端（ Client ）真实信息，包括客户端 IP（ cip ）、客户端端口（ cport ）。
Client 在 ksc 之外，通过 LB 访问 RS，RS 可为虚机或物理机。
Client 与 RS 同属 ksc，可通过 LB 访问 RS，RS 可为云服务器或裸金属服务器。
Client 通过其 ipv6 地址（cip6）访问 vip6:vport 服务，LB与 RS 通信为 ipv4 地址。Option 字段携带 cip6:cport 信息。
Client 通过其 ipv4 地址（ cip4 ）访问 vip4:vport 服务，LB与 RS 通信为 ipv4 地址。Option 字段携带 cip4:cport 信息。
在 fullnat（ fnat ）/ttm_cip 模式下，负载均衡（ LB ）向后端 RS 转发报文时，会以其 local ip（lip）作为报文源 ip。即，常规模式下在 RS 端得不到 client 端真实 ip/port 信息。
在 tcp 建立连接阶段，ttm 通过 tcp 握手的第三个报文（ack）内利用 tcp option 字段加入 client ip/port 信息，实现在 RS 端的信息获取。Option 字段由 ksc-gw 封装进报文，由加载 kgwttm 的rs 端解封装。Option 字段格式如下所示：
/* 自定义 client 信息结构体 */
struct ttm_peer {
__u16 af;
__be16 port;
union {
__u32 all[4];
__be32 ip;
__be32 ip6[4];
struct in_addr in;
struct in6_addr in6;
};
};源码下载链接(4to4和6to4场景均支持)
支持内核版本：v2.6* 、v3.* 、v4.*(特定版本环境可提供技术支持)
内核环境准备
# rpm -qa | grep kernel-devel-$(uname -r)
# yum install "kernel-devel-$(uname -r)"解压 kgwttm_ipv4.zip
# tar -zxf kgwttm_ipv4.zip编译
# cd kgwttm
# ./build.sh安装
# sudo rpm -ivh kmod-kgwttm-v.*.rpm检查是否安装成功
#lsmod | grep kgwttm以 python 为例：
RS 端，用户在 tcp socket server 端 accept()返回成功后，通过调用 socket.getsockopt(level,optname[, buflen])，其中 level 赋值 socket.SOL_IP, optname 赋值 1345，buflen 赋值 20。获取返回信息后，按照 struct ttm_data_v6 解析相应返回值即可。
在 6to4 以及 4to4 模式中，我们提供socket.getsockopt()指定 optname 为 1345 方式获取 client端信息。
4to4 场景，还可通过 socket.accept()返回参数或调用 socket.getpeername()获取 client 端信息。
具体样例参考附录
官网下载nginx Code
解压进入目录，configure 配置 ./configure --prefix=/usr/local/nginx --without-http_rewrite_module --add-module=…/ngx_http_realipv6_module-master-6d00184751fea369efcda8e8b70306bfaf43d991/ --without-http_gzip_module （模块源码下载）
注：配置阶段需要加载一个解析模块，用于获取前端网关在option中传送过来的真实源IPv6 地址
//ngx_http_realipv6_module.c
#include <ngx_config.h>
#include <ngx_core.h>
#include <ngx_http.h>
#include <linux/types.h>
#include <errno.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#define NGX_TTM_BASE_CTL (64 + 1024 + 64 + 64 + 64 + 64) /* base */
#define NGX_TTM_SO_GET_PEER (NGX_TTM_BASE_CTL + 1)
typedef struct ttm_peer_s ttm_peer_t;
struct ttm_peer_s
{
__u16 af;
__be16 port;
union {
__u32 all[4];
__be32 ip;
__be32 ip6[4];
struct in_addr in;
struct in6_addr in6;
};
};
typedef struct
{
ngx_str_t header;
ngx_flag_t set_real_ipv6;
} ngx_http_realipv6_loc_conf_t;
typedef struct
{
ngx_str_t remote_ipv6_addr;
} ngx_http_realipv6_ctx_t;
static ngx_int_t ngx_http_realipv6_handler(ngx_http_request_t *r);
static void ngx_http_realipv6_cleanup(void *data);
static void *ngx_http_realipv6_create_loc_conf(ngx_conf_t *cf);
static ngx_int_t ngx_http_realipv6_add_variables(ngx_conf_t *cf);
static ngx_int_t ngx_http_realipv6_init(ngx_conf_t *cf);
static ngx_int_t ngx_http_realipv6_remote_addr_variable(ngx_http_request_t *r,
ngx_http_variable_value_t *v, uintptr_t data);
static ngx_command_t ngx_http_realipv6_commands[] = {
{ngx_string("set_real_ipv6_header"),
NGX_HTTP_MAIN_CONF | NGX_HTTP_SRV_CONF | NGX_HTTP_LOC_CONF | NGX_CONF_TAKE1,
ngx_conf_set_str_slot,
NGX_HTTP_LOC_CONF_OFFSET,
offsetof(ngx_http_realipv6_loc_conf_t, header),
NULL},
{ngx_string("set_real_ipv6"),
NGX_HTTP_MAIN_CONF | NGX_HTTP_SRV_CONF | NGX_HTTP_LOC_CONF | NGX_CONF_FLAG,
ngx_conf_set_flag_slot,
NGX_HTTP_LOC_CONF_OFFSET,
offsetof(ngx_http_realipv6_loc_conf_t, set_real_ipv6),
NULL},
ngx_null_command};
static ngx_http_module_t ngx_http_realipv6_module_ctx = {
ngx_http_realipv6_add_variables, /* preconfiguration */
ngx_http_realipv6_init, /* postconfiguration */
// ngx_http_realipv6_add_variables, /* create main configuration */
NULL,
NULL, /* init main configuration */
NULL, /* create server configuration */
NULL, /* merge server configuration */
ngx_http_realipv6_create_loc_conf, /* create location configuration */
NULL /* merge location configuration */
};
ngx_module_t ngx_http_realipv6_module = {
NGX_MODULE_V1,
&ngx_http_realipv6_module_ctx, /* module context */
ngx_http_realipv6_commands, /* module directives */
NGX_HTTP_MODULE, /* module type */
NULL, /* init master */
NULL, /* init module */
NULL, /* init process */
NULL, /* init thread */
NULL, /* exit thread */
NULL, /* exit process */
NULL, /* exit master */
NGX_MODULE_V1_PADDING};
static ngx_http_variable_t ngx_http_realipv6_vars[] = {
{ngx_string("x_real_ipv6"), NULL,
ngx_http_realipv6_remote_addr_variable, 0, 0, 0},
{ngx_null_string, NULL, NULL, 0, 0, 0}};
static ngx_int_t
ngx_http_realipv6_handler(ngx_http_request_t *r)
{
u_char *p;
socklen_t len;
size_t plen;
ngx_connection_t *c;
ngx_http_realipv6_loc_conf_t *rlcf;
ngx_pool_cleanup_t *cln;
ngx_http_realipv6_ctx_t *ctx;
rlcf = ngx_http_get_module_loc_conf(r, ngx_http_realipv6_module);
if (!rlcf->set_real_ipv6)
{
return NGX_DECLINED;
}
c = r->connection;
ttm_peer_t peer;
len = sizeof(peer);
if (getsockopt(c->fd, IPPROTO_IP, NGX_TTM_SO_GET_PEER, &peer, &len) < 0)
{
ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
"nginx getsockopt failed");
return NGX_DECLINED;
}
char addr[NGX_SOCKADDR_STRLEN];
const char *ret = inet_ntop(peer.af, &peer.all, addr, sizeof(addr));
if (ret == NULL)
{
ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
"nginx inet_ntop failed");
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
cln = ngx_pool_cleanup_add(r->pool, sizeof(ngx_http_realipv6_ctx_t));
if (cln == NULL)
{
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
ctx = cln->data;
ngx_http_set_ctx(r, ctx, ngx_http_realipv6_module);
c = r->connection;
plen = ngx_strlen(ret);
p = ngx_pnalloc(c->pool, plen);
if (p == NULL)
{
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
ngx_memcpy(p, ret, plen);
cln->handler = ngx_http_realipv6_cleanup;
ctx->remote_ipv6_addr.len = plen;
ctx->remote_ipv6_addr.data = p;
return NGX_DECLINED;
}
static void
ngx_http_realipv6_cleanup(void *data)
{
return;
}
static void *
ngx_http_realipv6_create_loc_conf(ngx_conf_t *cf)
{
ngx_http_realipv6_loc_conf_t *conf;
conf = ngx_pcalloc(cf->pool, sizeof(ngx_http_realipv6_loc_conf_t));
if (conf == NULL)
{
return NULL;
}
ngx_str_null(&conf->header);
conf->set_real_ipv6 = NGX_CONF_UNSET;
return conf;
}
static ngx_int_t
ngx_http_realipv6_add_variables(ngx_conf_t *cf)
{
ngx_http_variable_t *var, *v;
for (v = ngx_http_realipv6_vars; v->name.len; v++)
{
var = ngx_http_add_variable(cf, &v->name, v->flags);
if (var == NULL)
{
return NGX_ERROR;
}
var->get_handler = v->get_handler;
var->data = v->data;
}
return NGX_OK;
}
static ngx_int_t
ngx_http_realipv6_init(ngx_conf_t *cf)
{
ngx_http_handler_pt *h;
ngx_http_core_main_conf_t *cmcf;
cmcf = ngx_http_conf_get_module_main_conf(cf, ngx_http_core_module);
h = ngx_array_push(&cmcf->phases[NGX_HTTP_POST_READ_PHASE].handlers);
if (h == NULL)
{
return NGX_ERROR;
}
*h = ngx_http_realipv6_handler;
return NGX_OK;
}
static ngx_int_t
ngx_http_realipv6_remote_addr_variable(ngx_http_request_t *r,
ngx_http_variable_value_t *v, uintptr_t data)
{
ngx_str_t *remote_ipv6_addr;
ngx_pool_cleanup_t *cln;
ngx_http_realipv6_ctx_t *ctx;
ctx = ngx_http_get_module_ctx(r, ngx_http_realipv6_module);
if (ctx == NULL && (r->internal || r->filter_finalize))
{
/*
* if module context was reset, the original address
* can still be found in the cleanup handler
*/
for (cln = r->pool->cleanup; cln; cln = cln->next)
{
if (cln->handler == ngx_http_realipv6_cleanup)
{
ctx = cln->data;
break;
}
}
}
// remote_ipv6_addr = ctx->remote_ipv6_addr;
remote_ipv6_addr = ctx ? &ctx->remote_ipv6_addr : &r->connection->addr_text;
v->len = remote_ipv6_addr->len;
v->valid = 1;
v->no_cacheable = 0;
v->not_found = 0;
v->data = remote_ipv6_addr->data;
return NGX_OK;
}make&& make install
将编译好的nginx 更新到/usr/sbin/目录下
调整配置文件nginx.conf
//nginx.conf
#user nobody;
worker_processes 1;
#error_log logs/error.log;
#error_log logs/error.log notice;
#error_log logs/error.log info;
#pid logs/nginx.pid;
events {
worker_connections 1024;
}
http {
include mime.types;
default_type application/octet-stream;
log_format main '$x_real_ipv6 $remote_addr - $remote_user [$time_local] "$request" ' //默认这两行是注释掉，去掉注释。添加Log输出变量 x_real_ipv6
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main; //打开log文件实时输出log 日志
sendfile on;
#tcp_nopush on;
#keepalive_timeout 0;
keepalive_timeout 65;
#gzip on;
server {
listen 80;
server_name localhost;
#charset koi8-r;
#access_log logs/host.access.log main;
location / {
root html;
index index.html index.htm;
}
#error_page 404 /404.html;
# redirect server error pages to the static page /50x.html
#
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root html;
}
# proxy the PHP scripts to Apache listening on 127.0.0.1:80
#
#location ~ \.php$ {
# proxy_pass http://127.0.0.1;
#}
# pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
#
#location ~ \.php$ {
# root html;
# fastcgi_pass 127.0.0.1:9000;
# fastcgi_index index.php;
# fastcgi_param SCRIPT_FILENAME /scripts$fastcgi_script_name;
# include fastcgi_params;
#}
# deny access to .htaccess files, if Apache's document root
# concurs with nginx's one
#
#location ~ /\.ht {
# deny all;
#}
}
# another virtual host using mix of IP-, name-, and port-based configuration
#
#server {
# listen 8000;
# listen somename:8080;
# server_name somename alias another.alias;
# location / {
# root html;
# index index.html index.htm;
# }
#}
# HTTPS server
#
#server {
# listen 443 ssl;
# server_name localhost;
# ssl_certificate cert.pem;
# ssl_certificate_key cert.key;
# ssl_session_cache shared:SSL:1m;
# ssl_session_timeout 5m;
# ssl_ciphers HIGH:!aNULL:!MD5;
# ssl_prefer_server_ciphers on;
# location / {
# root html;
# index index.html index.htm;
# }
#}
}
检查 rpm 包安装情况
# rpm –qa|grep kgwttm卸载
# rpm -e `rpm -qa|grep ttm`再次检查
# lsmod|grep kgwttm用户安装 kgwttm 模块后，可直接运行源码包内 example_ipv6/ 目录下的 c/python server 端 demo 实例。基于业务场景提供 client 端 tcp 访问，在 RS 侧观察效果。
… … # 用户 socket 逻辑
ss, addr = s.accept()
# 4to4 场景，可直接获得 cip/cport 信息
# 6to4 场景，获得 xgw 代理 lip/lport 信息
print 'get client info:', addr # 4to4 方法一
print 'getpeername:', ss.getpeername() # 4to4 方法二
# 0 == SOL_IP; 1345 == TTM_SO_GET_PEER; 20 == buff len
str = ss.getsockopt(0, 1345, 20) # 4to4 方法三 / 6to4 方法
# 按照 struct ttm_data_v6 格式解析
obj_p = struct.Struct('!bbH4s12s')
data = obj_p.unpack(str)
# 根据协议簇解析对应信。2 == AF_INET， 10 == AF_INET6
# 4to4 场景推荐使用上述两种方式获取 client 信息
print 'port:', data[2]
if(data[0] == 2):
print 'ipv4:', inet_ntop(data[0], data[3])
else:
print 'ipv6:', inet_ntop(data[0], data[3]+data[4]) # 10 is AF_INET6
… … # 用户后续逻辑#define TTM_BASE_CTL (64+1024+64+64+64+64) /* base */
#define TTM_SO_GET_PEER (TTM_BASE_CTL+1) /* 1345, 自定义 id */
/* 自定义 client 信息结构体 */
struct ttm_peer {
__u16 af;
__be16 port;
union {
__u32 all[4];
__be32 ip;
__be32 ip6[4];
struct in_addr in;
struct in6_addr in6;
};
};
… … //用户 socket 逻辑
new_fd=accept(sockfd,(struct sockaddr *)&client_addr, &sin_size);
/* 4to4 场景，可直接获得 cip/cport 信息 */
fprintf(stderr,"Server get connection from %s:%u\n",
inet_ntoa(client_addr.sin_addr),
ntohs(client_addr.sin_port));
/* 4to4 或 6to4 获取 client 信息 */
if (getsockopt(new_fd, IPPROTO_IP, TTM_SO_GET_PEER, &peer, &len) < 0) {
fprintf(stderr, "getsockopt failed\n");
} else {
inet_ntop(peer.af, &peer.all, &addr_string, sizeof(addr_string));
printf("getsockopt success(addr:%s port:%u)\n", addr_string, ntohs(peer.port));
}纯净模式