udp2raw/server.cpp

/*
 * server.cpp
 *
 *  Created on: Aug 29, 2018
 *      Author: root
 */

#ifndef UDP2RAW_MP

#include "common.h"
#include "network.h"
#include "connection.h"
#include "misc.h"
#include "log.h"
#include "lib/md5.h"
#include "encrypt.h"
#include "fd_manager.h"

int server_on_timer_multi(conn_info_t &conn_info)  // for server. called when a timer is ready in epoll.for server,there will be one timer for every connection
// there is also a global timer for server,but its not handled here
{
    char ip_port[max_addr_len];
    // u32_t ip=conn_info.raw_info.send_info.dst_ip;
    // u32_t port=conn_info.raw_info.send_info.dst_port;

    address_t tmp_addr;
    tmp_addr.from_ip_port_new(raw_ip_version, &conn_info.raw_info.send_info.new_dst_ip, conn_info.raw_info.send_info.dst_port);
    // sprintf(ip_port,"%s:%d",my_ntoa(ip),port);
    tmp_addr.to_str(ip_port);

    // keep_iptables_rule();
    mylog(log_trace, "server timer!\n");
    raw_info_t &raw_info = conn_info.raw_info;

    assert(conn_info.state.server_current_state == server_ready);

    if (conn_info.state.server_current_state == server_ready) {
        conn_info.blob->conv_manager.s.clear_inactive(ip_port);
        /*
        if( get_current_time()-conn_info.last_hb_recv_time>heartbeat_timeout )
        {
                mylog(log_trace,"%lld %lld\n",get_current_time(),conn_info.last_state_time);
                conn_info.server_current_state=server_nothing;

                //conn_manager.current_ready_ip=0;
                //conn_manager.current_ready_port=0;

                mylog(log_info,"changed state to server_nothing\n");
                return 0;
        }*/
        // dont need to do this at server,conn_manger will clear expired connections

        if (get_current_time() - conn_info.last_hb_sent_time < heartbeat_interval) {
            return 0;
        }

        if (hb_mode == 0)
            send_safer(conn_info, 'h', hb_buf, 0);  /////////////send
        else
            send_safer(conn_info, 'h', hb_buf, hb_len);
        conn_info.last_hb_sent_time = get_current_time();

        mylog(log_debug, "heart beat sent<%x,%x>\n", conn_info.my_id, conn_info.oppsite_id);
    } else {
        mylog(log_fatal, "this shouldnt happen!\n");
        myexit(-1);
    }
    return 0;
}
int server_on_raw_recv_ready(conn_info_t &conn_info, char *ip_port, char type, char *data, int data_len)  // called while the state for a connection is server_ready
// receives data and heart beat by recv_safer.
{
    raw_info_t &raw_info = conn_info.raw_info;
    packet_info_t &send_info = conn_info.raw_info.send_info;
    packet_info_t &recv_info = conn_info.raw_info.recv_info;
    // char ip_port[40];

    // sprintf(ip_port,"%s:%d",my_ntoa(recv_info.src_ip),recv_info.src_port);

    /*
            if (recv_info.src_ip != send_info.dst_ip
                            || recv_info.src_port != send_info.dst_port) {
                    mylog(log_debug, "unexpected adress\n");
                    return 0;
            }*/

    if (type == 'h' && data_len >= 0) {
        // u32_t tmp = ntohl(*((u32_t *) &data[sizeof(u32_t)]));
        mylog(log_debug, "[%s][hb]received hb \n", ip_port);
        conn_info.last_hb_recv_time = get_current_time();
        return 0;
    } else if (type == 'd' && data_len >= int(sizeof(u32_t))) {
        // u32_t tmp_conv_id = ntohl(*((u32_t *) &data[0]));
        my_id_t tmp_conv_id;
        memcpy(&tmp_conv_id, &data[0], sizeof(tmp_conv_id));
        tmp_conv_id = ntohl(tmp_conv_id);

        if (hb_mode == 0)
            conn_info.last_hb_recv_time = get_current_time();

        mylog(log_trace, "conv:%u\n", tmp_conv_id);
        if (!conn_info.blob->conv_manager.s.is_conv_used(tmp_conv_id)) {
            if (conn_info.blob->conv_manager.s.get_size() >= max_conv_num) {
                mylog(log_warn,
                      "[%s]ignored new conv %x connect bc max_conv_num exceed\n", ip_port,
                      tmp_conv_id);
                return 0;
            }

            /*
            struct sockaddr_in remote_addr_in={0};

            socklen_t slen = sizeof(sockaddr_in);
            //memset(&remote_addr_in, 0, sizeof(remote_addr_in));
            remote_addr_in.sin_family = AF_INET;
            remote_addr_in.sin_port = htons(remote_port);
            remote_addr_in.sin_addr.s_addr = remote_ip_uint32;



            int new_udp_fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);


            if (new_udp_fd < 0) {
                    mylog(log_warn, "[%s]create udp_fd error\n",ip_port);
                    return -1;
            }
            setnonblocking(new_udp_fd);
            set_buf_size(new_udp_fd,socket_buf_size);

            mylog(log_debug, "[%s]created new udp_fd %d\n",ip_port, new_udp_fd);
            int ret = connect(new_udp_fd, (struct sockaddr *) &remote_addr_in,
                            slen);
            if (ret != 0) {
                    mylog(log_warn, "udp fd connect fail\n");
                    close(new_udp_fd);
                    return -1;
            }*/

            int new_udp_fd = remote_addr.new_connected_udp_fd();
            if (new_udp_fd < 0) {
                mylog(log_warn, "[%s]new_connected_udp_fd() failed\n", ip_port);
                return -1;
            }

            struct epoll_event ev;

            fd64_t new_udp_fd64 = fd_manager.create(new_udp_fd);
            fd_manager.get_info(new_udp_fd64).p_conn_info = &conn_info;

            mylog(log_trace, "[%s]u64: %lld\n", ip_port, new_udp_fd64);
            ev.events = EPOLLIN;

            ev.data.u64 = new_udp_fd64;

            int ret = epoll_ctl(epollfd, EPOLL_CTL_ADD, new_udp_fd, &ev);

            if (ret != 0) {
                mylog(log_warn, "[%s]add udp_fd error\n", ip_port);
                close(new_udp_fd);
                return -1;
            }

            conn_info.blob->conv_manager.s.insert_conv(tmp_conv_id, new_udp_fd64);

            // assert(conn_manager.udp_fd_mp.find(new_udp_fd)==conn_manager.udp_fd_mp.end());

            // conn_manager.udp_fd_mp[new_udp_fd] = &conn_info;

            // pack_u64(conn_info.raw_info.recv_info.src_ip,conn_info.raw_info.recv_info.src_port);

            mylog(log_info, "[%s]new conv conv_id=%x, assigned fd=%d\n", ip_port,
                  tmp_conv_id, new_udp_fd);
        }

        fd64_t fd64 = conn_info.blob->conv_manager.s.find_data_by_conv(tmp_conv_id);

        conn_info.blob->conv_manager.s.update_active_time(tmp_conv_id);

        int fd = fd_manager.to_fd(fd64);

        mylog(log_trace, "[%s]received a data from fake tcp,len:%d\n", ip_port, data_len);
        int ret = send(fd, data + sizeof(u32_t),
                       data_len - (sizeof(u32_t)), 0);

        mylog(log_trace, "[%s]%d byte sent  ,fd :%d\n ", ip_port, ret, fd);
        if (ret < 0) {
            mylog(log_warn, "send returned %d\n", ret);
            // perror("what happened????");
        }
        return 0;
    }
    return 0;
}

int server_on_raw_recv_pre_ready(conn_info_t &conn_info, char *ip_port, u32_t tmp_oppsite_const_id)  // do prepare work before state change to server ready for a specifc connection
// connection recovery is also handle here
{
    // u32_t ip;uint16_t port;
    // ip=conn_info.raw_info.recv_info.src_ip;
    // port=conn_info.raw_info.recv_info.src_port;
    // char ip_port[40];
    // sprintf(ip_port,"%s:%d",my_ntoa(ip),port);

    mylog(log_info, "[%s]received handshake oppsite_id:%x  my_id:%x\n", ip_port, conn_info.oppsite_id, conn_info.my_id);

    mylog(log_info, "[%s]oppsite const_id:%x \n", ip_port, tmp_oppsite_const_id);
    if (conn_manager.const_id_mp.find(tmp_oppsite_const_id) == conn_manager.const_id_mp.end()) {
        // conn_manager.const_id_mp=

        if (conn_manager.ready_num >= max_ready_conn_num) {
            mylog(log_info, "[%s]max_ready_conn_num,cant turn to ready\n", ip_port);
            conn_info.state.server_current_state = server_idle;
            return 0;
        }

        conn_info.prepare();
        conn_info.state.server_current_state = server_ready;
        conn_info.oppsite_const_id = tmp_oppsite_const_id;
        conn_manager.ready_num++;
        conn_manager.const_id_mp[tmp_oppsite_const_id] = &conn_info;

        // conn_info.last_state_time=get_current_time(); //dont change this!!!!!!!!!!!!!!!!!!!!!!!!!

        // conn_manager.current_ready_ip=ip;
        // conn_manager.current_ready_port=port;

        // my_id=conn_info.my_id;
        // oppsite_id=conn_info.oppsite_id;
        conn_info.last_hb_recv_time = get_current_time();

        conn_info.last_hb_sent_time = conn_info.last_hb_recv_time;  //=get_current_time()

        if (hb_mode == 0)
            send_safer(conn_info, 'h', hb_buf, 0);  /////////////send
        else
            send_safer(conn_info, 'h', hb_buf, hb_len);

        mylog(log_info, "[%s]changed state to server_ready\n", ip_port);
        conn_info.blob->anti_replay.re_init();

        // g_conn_info=conn_info;
        int new_timer_fd;
        set_timer_server(epollfd, new_timer_fd, conn_info.timer_fd64);

        fd_manager.get_info(conn_info.timer_fd64).p_conn_info = &conn_info;
        // assert(conn_manager.timer_fd_mp.find(new_timer_fd)==conn_manager.timer_fd_mp.end());
        // conn_manager.timer_fd_mp[new_timer_fd] = &conn_info;//pack_u64(ip,port);

        // timer_fd_mp[new_timer_fd]
        /*
         if(oppsite_const_id!=0&&tmp_oppsite_const_id!=oppsite_const_id)  //TODO MOVE TO READY
         {
         mylog(log_info,"cleared all conv bc of const id doesnt match\n");
         conv_manager.clear();
         }*/
        // oppsite_const_id=tmp_oppsite_const_id;
    } else {
        conn_info_t &ori_conn_info = *conn_manager.const_id_mp[tmp_oppsite_const_id];

        if (ori_conn_info.state.server_current_state == server_ready) {
            if (conn_info.last_state_time < ori_conn_info.last_state_time) {
                mylog(log_info, "[%s]conn_info.last_state_time<ori_conn_info.last_state_time. ignored new handshake\n", ip_port);
                conn_info.state.server_current_state = server_idle;
                conn_info.oppsite_const_id = 0;
                return 0;
            }
            address_t addr1;
            addr1.from_ip_port_new(raw_ip_version, &ori_conn_info.raw_info.recv_info.new_src_ip, ori_conn_info.raw_info.recv_info.src_port);
            if (!conn_manager.exist(addr1))  // TODO remove this
            {
                mylog(log_fatal, "[%s]this shouldnt happen\n", ip_port);
                myexit(-1);
            }
            address_t addr2;
            addr2.from_ip_port_new(raw_ip_version, &conn_info.raw_info.recv_info.new_src_ip, conn_info.raw_info.recv_info.src_port);
            if (!conn_manager.exist(addr2))  // TODO remove this
            {
                mylog(log_fatal, "[%s]this shouldnt happen2\n", ip_port);
                myexit(-1);
            }
            conn_info_t *&p_ori = conn_manager.find_insert_p(addr1);
            conn_info_t *&p = conn_manager.find_insert_p(addr2);
            conn_info_t *tmp = p;
            p = p_ori;
            p_ori = tmp;

            mylog(log_info, "[%s]grabbed a connection\n", ip_port);

            // ori_conn_info.state.server_current_state=server_ready;
            ori_conn_info.recover(conn_info);

            // send_safer(ori_conn_info, 'h',hb_buf, hb_len);
            // ori_conn_info.blob->anti_replay.re_init();
            if (hb_mode == 0)
                send_safer(ori_conn_info, 'h', hb_buf, 0);  /////////////send
            else
                send_safer(ori_conn_info, 'h', hb_buf, hb_len);

            ori_conn_info.last_hb_recv_time = get_current_time();

            conn_info.state.server_current_state = server_idle;
            conn_info.oppsite_const_id = 0;

        } else {
            mylog(log_fatal, "[%s]this should never happen\n", ip_port);
            myexit(-1);
        }
        return 0;
    }
    return 0;
}
int server_on_raw_recv_handshake1(conn_info_t &conn_info, char *ip_port, char *data, int data_len)  // called when server received a handshake1 packet from client
{
    packet_info_t &send_info = conn_info.raw_info.send_info;
    packet_info_t &recv_info = conn_info.raw_info.recv_info;
    raw_info_t &raw_info = conn_info.raw_info;

    // u32_t ip=conn_info.raw_info.recv_info.src_ip;
    // uint16_t port=conn_info.raw_info.recv_info.src_port;

    // char ip_port[40];
    // sprintf(ip_port,"%s:%d",my_ntoa(ip),port);

    if (data_len < int(3 * sizeof(my_id_t))) {
        mylog(log_debug, "[%s] data_len=%d too short to be a handshake\n", ip_port, data_len);
        return -1;
    }
    // id_t tmp_oppsite_id=  ntohl(* ((u32_t *)&data[0]));
    my_id_t tmp_oppsite_id;
    memcpy(&tmp_oppsite_id, (u32_t *)&data[0], sizeof(tmp_oppsite_id));
    tmp_oppsite_id = ntohl(tmp_oppsite_id);

    // id_t tmp_my_id=ntohl(* ((u32_t *)&data[sizeof(id_t)]));
    my_id_t tmp_my_id;
    memcpy(&tmp_my_id, &data[sizeof(my_id_t)], sizeof(tmp_my_id));
    tmp_my_id = ntohl(tmp_my_id);

    if (tmp_my_id == 0)  // received  init handshake again
    {
        if (raw_mode == mode_faketcp) {
            send_info.seq = recv_info.ack_seq;
            send_info.ack_seq = recv_info.seq + raw_info.recv_info.data_len;
            send_info.ts_ack = recv_info.ts;
        }
        if (raw_mode == mode_icmp) {
            send_info.my_icmp_seq = recv_info.my_icmp_seq;
        }
        send_handshake(raw_info, conn_info.my_id, tmp_oppsite_id, const_id);  //////////////send

        mylog(log_info, "[%s]changed state to server_handshake1,my_id is %x\n", ip_port, conn_info.my_id);
    } else if (tmp_my_id == conn_info.my_id) {
        conn_info.oppsite_id = tmp_oppsite_id;
        // id_t tmp_oppsite_const_id=ntohl(* ((u32_t *)&data[sizeof(id_t)*2]));

        my_id_t tmp_oppsite_const_id;
        memcpy(&tmp_oppsite_const_id, &data[sizeof(my_id_t) * 2], sizeof(tmp_oppsite_const_id));
        tmp_oppsite_const_id = ntohl(tmp_oppsite_const_id);

        if (raw_mode == mode_faketcp) {
            send_info.seq = recv_info.ack_seq;
            send_info.ack_seq = recv_info.seq + raw_info.recv_info.data_len;
            send_info.ts_ack = recv_info.ts;
        }

        if (raw_mode == mode_icmp) {
            send_info.my_icmp_seq = recv_info.my_icmp_seq;
        }

        server_on_raw_recv_pre_ready(conn_info, ip_port, tmp_oppsite_const_id);

    } else {
        mylog(log_debug, "[%s]invalid my_id %x,my_id is %x\n", ip_port, tmp_my_id, conn_info.my_id);
    }
    return 0;
}
int server_on_recv_safer_multi(conn_info_t &conn_info, char type, char *data, int data_len) {
    return 0;
}
int server_on_raw_recv_multi()  // called when server received an raw packet
{
    char dummy_buf[buf_len];
    raw_info_t peek_raw_info;
    peek_raw_info.peek = 1;
    packet_info_t &peek_info = peek_raw_info.recv_info;
    mylog(log_trace, "got a packet\n");
    if (pre_recv_raw_packet() < 0) return -1;
    if (peek_raw(peek_raw_info) < 0) {
        discard_raw_packet();
        // recv(raw_recv_fd, 0,0, 0  );//
        // struct sockaddr saddr;
        // socklen_t saddr_size=sizeof(saddr);
        /// recvfrom(raw_recv_fd, 0,0, 0 ,&saddr , &saddr_size);//
        mylog(log_trace, "peek_raw failed\n");
        return -1;
    } else {
        mylog(log_trace, "peek_raw success\n");
    }
    // u32_t ip=peek_info.src_ip;uint16_t port=peek_info.src_port;

    int data_len;
    char *data;

    address_t addr;
    addr.from_ip_port_new(raw_ip_version, &peek_info.new_src_ip, peek_info.src_port);

    char ip_port[max_addr_len];
    addr.to_str(ip_port);
    // sprintf(ip_port,"%s:%d",my_ntoa(ip),port);
    mylog(log_trace, "[%s]peek_raw\n", ip_port);

    if (raw_mode == mode_faketcp && peek_info.syn == 1) {
        if (!conn_manager.exist(addr) || conn_manager.find_insert(addr).state.server_current_state != server_ready) {  // reply any syn ,before state become ready

            raw_info_t tmp_raw_info;
            if (recv_raw0(tmp_raw_info, data, data_len) < 0) {
                return 0;
            }
            if (data_len >= max_data_len + 1) {
                mylog(log_debug, "data_len=%d >= max_data_len+1,ignored", data_len);
                return -1;
            }
            if (use_tcp_dummy_socket != 0)
                return 0;
            raw_info_t &raw_info = tmp_raw_info;
            packet_info_t &send_info = raw_info.send_info;
            packet_info_t &recv_info = raw_info.recv_info;

            send_info.new_src_ip = recv_info.new_dst_ip;
            send_info.src_port = recv_info.dst_port;

            send_info.dst_port = recv_info.src_port;
            send_info.new_dst_ip = recv_info.new_src_ip;

            if (lower_level) {
                handle_lower_level(raw_info);
            }

            if (data_len == 0 && raw_info.recv_info.syn == 1 && raw_info.recv_info.ack == 0) {
                send_info.ack_seq = recv_info.seq + 1;

                send_info.psh = 0;
                send_info.syn = 1;
                send_info.ack = 1;
                send_info.ts_ack = recv_info.ts;

                mylog(log_info, "[%s]received syn,sent syn ack back\n", ip_port);
                send_raw0(raw_info, 0, 0);
                return 0;
            }
        } else {
            discard_raw_packet();
            // recv(raw_recv_fd, 0,0,0);
        }
        return 0;
    }
    if (!conn_manager.exist(addr)) {
        if (conn_manager.mp.size() >= max_handshake_conn_num) {
            mylog(log_info, "[%s]reached max_handshake_conn_num,ignored new handshake\n", ip_port);
            discard_raw_packet();
            // recv(raw_recv_fd, 0,0, 0  );//
            return 0;
        }

        raw_info_t tmp_raw_info;

        if (raw_mode == mode_icmp) {
            tmp_raw_info.send_info.dst_port = tmp_raw_info.send_info.src_port = addr.get_port();
        }
        if (recv_bare(tmp_raw_info, data, data_len) < 0) {
            return 0;
        }
        if (data_len < int(3 * sizeof(my_id_t))) {
            mylog(log_debug, "[%s]too short to be a handshake\n", ip_port);
            return -1;
        }

        // id_t zero=ntohl(* ((u32_t *)&data[sizeof(id_t)]));
        my_id_t zero;
        memcpy(&zero, &data[sizeof(my_id_t)], sizeof(zero));
        zero = ntohl(zero);

        if (zero != 0) {
            mylog(log_debug, "[%s]not a invalid initial handshake\n", ip_port);
            return -1;
        }

        mylog(log_info, "[%s]got packet from a new ip\n", ip_port);

        conn_info_t &conn_info = conn_manager.find_insert(addr);
        conn_info.raw_info = tmp_raw_info;
        raw_info_t &raw_info = conn_info.raw_info;

        packet_info_t &send_info = conn_info.raw_info.send_info;
        packet_info_t &recv_info = conn_info.raw_info.recv_info;

        // conn_info.ip_port.ip=ip;
        // conn_info.ip_port.port=port;

        send_info.new_src_ip = recv_info.new_dst_ip;
        send_info.src_port = recv_info.dst_port;

        send_info.dst_port = recv_info.src_port;
        send_info.new_dst_ip = recv_info.new_src_ip;

        if (lower_level) {
            handle_lower_level(raw_info);
        }

        // id_t tmp_oppsite_id=  ntohl(* ((u32_t *)&data[0]));
        // mylog(log_info,"[%s]handshake1 received %x\n",ip_port,tmp_oppsite_id);

        conn_info.my_id = get_true_random_number_nz();

        mylog(log_info, "[%s]created new conn,state: server_handshake1,my_id is %x\n", ip_port, conn_info.my_id);

        conn_info.state.server_current_state = server_handshake1;
        conn_info.last_state_time = get_current_time();

        server_on_raw_recv_handshake1(conn_info, ip_port, data, data_len);
        return 0;
    }

    conn_info_t &conn_info = conn_manager.find_insert(addr);  // insert if not exist
    packet_info_t &send_info = conn_info.raw_info.send_info;
    packet_info_t &recv_info = conn_info.raw_info.recv_info;
    raw_info_t &raw_info = conn_info.raw_info;

    if (conn_info.state.server_current_state == server_handshake1) {
        if (recv_bare(raw_info, data, data_len) != 0) {
            return -1;
        }
        return server_on_raw_recv_handshake1(conn_info, ip_port, data, data_len);
    }
    if (conn_info.state.server_current_state == server_ready) {
        vector<char> type_vec;
        vector<string> data_vec;
        recv_safer_multi(conn_info, type_vec, data_vec);
        if (data_vec.empty()) {
            mylog(log_debug, "recv_safer failed!\n");
            return -1;
        }

        for (int i = 0; i < (int)type_vec.size(); i++) {
            char type = type_vec[i];
            char *data = (char *)data_vec[i].c_str();  // be careful, do not append data to it
            int data_len = data_vec[i].length();
            server_on_raw_recv_ready(conn_info, ip_port, type, data, data_len);
        }
        return 0;
    }

    if (conn_info.state.server_current_state == server_idle) {
        discard_raw_packet();
        // recv(raw_recv_fd, 0,0, 0  );//
        return 0;
    }
    mylog(log_fatal, "we should never run to here\n");
    myexit(-1);
    return -1;
}

int server_on_udp_recv(conn_info_t &conn_info, fd64_t fd64) {
    char buf[buf_len];

    if (conn_info.state.server_current_state != server_ready)  // TODO remove this for peformance
    {
        mylog(log_fatal, "p_conn_info->state.server_current_state!=server_ready!!!this shouldnt happen\n");
        myexit(-1);
    }

    // conn_info_t &conn_info=*p_conn_info;

    assert(conn_info.blob->conv_manager.s.is_data_used(fd64));

    u32_t conv_id = conn_info.blob->conv_manager.s.find_conv_by_data(fd64);

    int fd = fd_manager.to_fd(fd64);

    int recv_len = recv(fd, buf, max_data_len + 1, 0);

    mylog(log_trace, "received a packet from udp_fd,len:%d\n", recv_len);

    if (recv_len == max_data_len + 1) {
        mylog(log_warn, "huge packet, data_len > %d,dropped\n", max_data_len);
        return -1;
    }

    if (recv_len < 0) {
        mylog(log_debug, "udp fd,recv_len<0 continue,%s\n", strerror(errno));
        return -1;
    }

    if (recv_len >= mtu_warn) {
        mylog(log_warn, "huge packet,data len=%d (>=%d).strongly suggested to set a smaller mtu at upper level,to get rid of this warn\n ", recv_len, mtu_warn);
    }

    // conn_info.conv_manager->update_active_time(conv_id);  server dosnt update from upd side,only update from raw side.  (client updates at both side)

    if (conn_info.state.server_current_state == server_ready) {
        send_data_safer(conn_info, buf, recv_len, conv_id);
        // send_data(g_packet_info_send,buf,recv_len,my_id,oppsite_id,conv_id);
        mylog(log_trace, "send_data_safer ,sent !!\n");
    }

    return 0;
}

int server_event_loop() {
    char buf[buf_len];

    int i, j, k;
    int ret;

    if (raw_ip_version == AF_INET) {
        if (local_addr.inner.ipv4.sin_addr.s_addr != 0) {
            bind_addr_used = 1;
            bind_addr.v4 = local_addr.inner.ipv4.sin_addr.s_addr;
        }
    } else {
        assert(raw_ip_version == AF_INET6);
        char zero_arr[16] = {0};
        if (memcmp(&local_addr.inner.ipv6.sin6_addr, zero_arr, 16) != 0) {
            bind_addr_used = 1;
            bind_addr.v6 = local_addr.inner.ipv6.sin6_addr;
        }
    }
    // bind_address_uint32=local_ip_uint32;//only server has bind adress,client sets it to zero

    if (lower_level) {
        if (lower_level_manual) {
            init_ifindex(if_name, raw_send_fd, ifindex);
            mylog(log_info, "we are running at lower-level (manual) mode\n");
        } else {
            mylog(log_info, "we are running at lower-level (auto) mode\n");
        }
    }

    if (raw_mode == mode_faketcp) {
        bind_fd = socket(local_addr.get_type(), SOCK_STREAM, 0);
    } else if (raw_mode == mode_udp || raw_mode == mode_icmp)  // bind an adress to avoid collision,for icmp,there is no port,just bind a udp port
    {
        bind_fd = socket(local_addr.get_type(), SOCK_DGRAM, 0);
    }

    // struct sockaddr_in temp_bind_addr={0};
    // bzero(&temp_bind_addr, sizeof(temp_bind_addr));

    // temp_bind_addr.sin_family = AF_INET;
    // temp_bind_addr.sin_port = local_addr.get_port();
    // temp_bind_addr.sin_addr.s_addr = local_addr.inner.ipv4.sin_addr.s_addr;

    if (bind(bind_fd, (struct sockaddr *)&local_addr.inner, local_addr.get_len()) != 0) {
        mylog(log_fatal, "bind fail\n");
        myexit(-1);
    }

    if (raw_mode == mode_faketcp) {
        if (listen(bind_fd, SOMAXCONN) != 0) {
            mylog(log_fatal, "listen fail\n");
            myexit(-1);
        }
    }

    // init_raw_socket();
    init_filter(local_addr.get_port());  // bpf filter

    epollfd = epoll_create1(0);
    const int max_events = 4096;

    struct epoll_event ev, events[max_events];
    if (epollfd < 0) {
        mylog(log_fatal, "epoll return %d\n", epollfd);
        myexit(-1);
    }

    ev.events = EPOLLIN;
    ev.data.u64 = raw_recv_fd;

    ret = epoll_ctl(epollfd, EPOLL_CTL_ADD, raw_recv_fd, &ev);
    if (ret != 0) {
        mylog(log_fatal, "add raw_fd error\n");
        myexit(-1);
    }
    int timer_fd;

    set_timer(epollfd, timer_fd);

    u64_t begin_time = 0;
    u64_t end_time = 0;

    mylog(log_info, "now listening at %s\n", local_addr.get_str());

    int fifo_fd = -1;

    if (fifo_file[0] != 0) {
        fifo_fd = create_fifo(fifo_file);
        ev.events = EPOLLIN;
        ev.data.u64 = fifo_fd;

        ret = epoll_ctl(epollfd, EPOLL_CTL_ADD, fifo_fd, &ev);
        if (ret != 0) {
            mylog(log_fatal, "add fifo_fd to epoll error %s\n", strerror(errno));
            myexit(-1);
        }
        mylog(log_info, "fifo_file=%s\n", fifo_file);
    }

    while (1)  ////////////////////////
    {
        if (about_to_exit) myexit(0);

        int nfds = epoll_wait(epollfd, events, max_events, 180 * 1000);
        if (nfds < 0) {  // allow zero
            if (errno == EINTR) {
                mylog(log_info, "epoll interrupted by signal,continue\n");
                // myexit(0);
            } else {
                mylog(log_fatal, "epoll_wait return %d,%s\n", nfds, strerror(errno));
                myexit(-1);
            }
        }
        int idx;
        for (idx = 0; idx < nfds; ++idx) {
            // mylog(log_debug,"ndfs:  %d \n",nfds);
            epoll_trigger_counter++;
            // printf("%d %d %d %d\n",timer_fd,raw_recv_fd,raw_send_fd,n);
            if ((events[idx].data.u64) == (u64_t)timer_fd) {
                if (debug_flag) begin_time = get_current_time();
                conn_manager.clear_inactive();
                u64_t dummy;
                int unused = read(timer_fd, &dummy, 8);
                // current_time_rough=get_current_time();
                if (debug_flag) {
                    end_time = get_current_time();
                    mylog(log_debug, "timer_fd,%llu,%llu,%llu\n", begin_time, end_time, end_time - begin_time);
                }

                mylog(log_trace, "epoll_trigger_counter:  %d \n", epoll_trigger_counter);
                epoll_trigger_counter = 0;

            } else if (events[idx].data.u64 == (u64_t)raw_recv_fd) {
                if (debug_flag) begin_time = get_current_time();
                server_on_raw_recv_multi();
                if (debug_flag) {
                    end_time = get_current_time();
                    mylog(log_debug, "raw_recv_fd,%llu,%llu,%llu  \n", begin_time, end_time, end_time - begin_time);
                }
            } else if (events[idx].data.u64 == (u64_t)fifo_fd) {
                int len = read(fifo_fd, buf, sizeof(buf));
                if (len < 0) {
                    mylog(log_warn, "fifo read failed len=%d,errno=%s\n", len, strerror(errno));
                    continue;
                }
                // assert(len>=0);
                buf[len] = 0;
                while (len >= 1 && buf[len - 1] == '\n')
                    buf[len - 1] = 0;
                mylog(log_info, "got data from fifo,len=%d,s=[%s]\n", len, buf);
                mylog(log_info, "unknown command\n");
            } else if (events[idx].data.u64 > u32_t(-1)) {
                fd64_t fd64 = events[idx].data.u64;
                if (!fd_manager.exist(fd64)) {
                    mylog(log_trace, "fd64 no longer exist\n");
                    return -1;
                }
                assert(fd_manager.exist_info(fd64));
                conn_info_t *p_conn_info = fd_manager.get_info(fd64).p_conn_info;
                conn_info_t &conn_info = *p_conn_info;
                if (fd64 == conn_info.timer_fd64)  //////////timer_fd64
                {
                    if (debug_flag) begin_time = get_current_time();
                    int fd = fd_manager.to_fd(fd64);
                    u64_t dummy;
                    int unused = read(fd, &dummy, 8);
                    assert(conn_info.state.server_current_state == server_ready);  // TODO remove this for peformance
                    server_on_timer_multi(conn_info);
                    if (debug_flag) {
                        end_time = get_current_time();
                        mylog(log_debug, "(events[idx].data.u64 >>32u) == 2u ,%llu,%llu,%llu  \n", begin_time, end_time, end_time - begin_time);
                    }
                } else  // udp_fd64
                {
                    if (debug_flag) begin_time = get_current_time();
                    server_on_udp_recv(conn_info, fd64);
                    if (debug_flag) {
                        end_time = get_current_time();
                        mylog(log_debug, "(events[idx].data.u64 >>32u) == 1u,%lld,%lld,%lld  \n", begin_time, end_time, end_time - begin_time);
                    }
                }
            } else {
                mylog(log_fatal, "unknown fd,this should never happen\n");
                myexit(-1);
            }
        }
    }
    return 0;
}

#endif