修改安卓端逻辑

2024-04-19 10:49:47 +08:00
parent 84824731a7
commit 8dfc3b8c43
26 changed files with 1191 additions and 186 deletions
@@ -31,7 +31,7 @@ openssl-sys = { git = "https://github.com/lbl8603/rust-openssl" ,optional = true
 libsm = {git="https://github.com/lbl8603/libsm" ,optional = true}

 mio = {version = "0.8.10",features = ["os-poll","net"]}
-
+crossbeam-queue = "0.3.11"
 [target.'cfg(target_os = "windows")'.dependencies]
 libloading = "0.8.0"

@@ -7,7 +7,7 @@ use std::time::Duration;
 use crossbeam_utils::atomic::AtomicCell;
 use parking_lot::{Mutex, RwLock};
 use rand::Rng;
-
+#[cfg(not(target_os = "android"))]
 use tun::device::IFace;

 use crate::channel::context::Context;
@@ -22,14 +22,15 @@ use crate::external_route::{AllowExternalRoute, ExternalRoute};
 use crate::handle::handshaker::Handshake;
 use crate::handle::maintain::PunchReceiver;
 use crate::handle::recv_data::RecvDataHandler;
-use crate::handle::{
-    maintain, tun_tap, BaseConfigInfo, ConnectStatus, CurrentDeviceInfo, PeerDeviceInfo,
-};
+use crate::handle::{maintain, BaseConfigInfo, ConnectStatus, CurrentDeviceInfo, PeerDeviceInfo};
 use crate::nat::NatTest;
+use crate::tun_tap_device::tun_create_helper::{DeviceAdapter, TunDeviceHelper};
 use crate::util::{
    Scheduler, SingleU64Adder, StopManager, U64Adder, WatchSingleU64Adder, WatchU64Adder,
 };
-use crate::{nat, tun_tap_device, DeviceInfo, VntCallback};
+use crate::{nat, VntCallback};
+#[cfg(not(target_os = "android"))]
+use crate::{tun_tap_device, DeviceInfo};

 #[derive(Clone)]
 pub struct Vnt {
@@ -113,10 +114,14 @@ impl Vnt {
            tcp_port,
        );

-        // 虚拟网卡
-        let device = tun_tap_device::create_device(&config)?;
-        let tun_info = DeviceInfo::new(device.name()?, device.version()?);
-        callback.create_tun(tun_info);
+        // pc上先创建虚拟网卡
+        #[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))]
+        let device = {
+            let device = tun_tap_device::create_device(&config)?;
+            let tun_info = DeviceInfo::new(device.name()?, device.version()?);
+            callback.create_tun(tun_info);
+            device
+        };
        // 服务停止管理器
        let stop_manager = {
            let callback = callback.clone();
@@ -146,13 +151,33 @@ impl Vnt {
            U64Adder::with_capacity(config.ports.as_ref().map(|v| v.len()).unwrap_or_default() + 8);
        let down_count_watcher = down_counter.watch();
        let handshake = Handshake::new(rsa_cipher.clone());
+        let up_counter = SingleU64Adder::new();
+        let up_count_watcher = up_counter.watch();
+        let tun_helper = TunDeviceHelper::new(
+            stop_manager.clone(),
+            context.clone(),
+            current_device.clone(),
+            external_route.clone(),
+            #[cfg(feature = "ip_proxy")]
+            proxy_map.clone(),
+            client_cipher.clone(),
+            server_cipher.clone(),
+            config.parallel,
+            up_counter,
+            device_list.clone(),
+        );
+        #[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))]
+        let device_adapter = DeviceAdapter::new(device.clone());
+        #[cfg(target_os = "android")]
+        let device_adapter = DeviceAdapter::new(tun_helper);
+
        let handler = RecvDataHandler::new(
            #[cfg(feature = "server_encrypt")]
            rsa_cipher,
            server_cipher.clone(),
            client_cipher.clone(),
            current_device.clone(),
-            device.clone(),
+            device_adapter,
            device_list.clone(),
            config_info.clone(),
            nat_test.clone(),
@@ -177,22 +202,10 @@ impl Vnt {
            config.tcp,
            tcp_socket_sender.clone(),
        );
-        let up_counter = SingleU64Adder::new();
-        let up_count_watcher = up_counter.watch();
-        tun_tap::tun_handler::start(
-            stop_manager.clone(),
-            context.clone(),
-            device.clone(),
-            current_device.clone(),
-            external_route,
-            #[cfg(feature = "ip_proxy")]
-            proxy_map,
-            client_cipher.clone(),
-            server_cipher.clone(),
-            config.parallel,
-            up_counter,
-            device_list.clone(),
-        )?;
+
+        #[cfg(not(target_os = "android"))]
+        tun_helper.start(device)?;
+
        maintain::idle_gateway(
            &scheduler,
            context.clone(),
@@ -36,8 +36,6 @@ pub struct Config {
    pub first_latency: bool,
    #[cfg(not(target_os = "android"))]
    pub device_name: Option<String>,
-    #[cfg(target_os = "android")]
-    pub device_fd: i32,
    pub use_channel_type: UseChannelType,
    //控制丢包率
    pub packet_loss_rate: Option<f64>,
@@ -68,7 +66,6 @@ impl Config {
        ports: Option<Vec<u16>>,
        first_latency: bool,
        #[cfg(not(target_os = "android"))] device_name: Option<String>,
-        #[cfg(target_os = "android")] device_fd: i32,
        use_channel_type: UseChannelType,
        packet_loss_rate: Option<f64>,
        packet_delay: u32,
@@ -113,8 +110,6 @@ impl Config {
            first_latency,
            #[cfg(not(target_os = "android"))]
            device_name,
-            #[cfg(target_os = "android")]
-            device_fd,
            use_channel_type,
            packet_loss_rate,
            packet_delay,
@@ -1,21 +1,25 @@
+use crate::handle::PeerDeviceStatus;
 #[cfg(feature = "server_encrypt")]
 use rsa::RsaPublicKey;
 use std::fmt::{Display, Formatter};
 use std::io;
 use std::net::{Ipv4Addr, SocketAddr};

+#[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))]
 #[derive(Debug)]
 pub struct DeviceInfo {
    pub name: String,
    pub version: String,
 }

+#[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))]
 impl Display for DeviceInfo {
    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
        f.write_str(&format!("name={} ,version={}", self.name, self.version))
    }
 }

+#[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))]
 impl DeviceInfo {
    pub fn new(name: String, version: String) -> Self {
        return Self { name, version };
@@ -68,6 +72,7 @@ impl Display for HandshakeInfo {
        f.write_str(&format!("server version={}", self.version))
    }
 }
+
 #[cfg(feature = "server_encrypt")]
 impl HandshakeInfo {
    pub fn new(public_key: RsaPublicKey, finger: String, version: String) -> Self {
@@ -85,6 +90,7 @@ impl HandshakeInfo {
        }
    }
 }
+
 #[cfg(not(feature = "server_encrypt"))]
 impl HandshakeInfo {
    pub fn new_no_secret(version: String) -> Self {
@@ -183,11 +189,89 @@ impl Into<u8> for ErrorType {
    }
 }

+#[cfg(target_os = "android")]
+#[derive(Debug)]
+pub struct DeviceConfig {
+    //本机虚拟IP
+    pub virtual_ip: Ipv4Addr,
+    //子网掩码
+    pub virtual_netmask: Ipv4Addr,
+    //虚拟网关
+    pub virtual_gateway: Ipv4Addr,
+    //虚拟网段
+    pub virtual_network: Ipv4Addr,
+    // 额外的路由
+    pub external_route: Vec<(Ipv4Addr, Ipv4Addr)>,
+}
+
+#[cfg(target_os = "android")]
+impl DeviceConfig {
+    pub fn new(
+        virtual_ip: Ipv4Addr,
+        virtual_netmask: Ipv4Addr,
+        virtual_gateway: Ipv4Addr,
+        virtual_network: Ipv4Addr,
+        external_route: Vec<(Ipv4Addr, Ipv4Addr)>,
+    ) -> Self {
+        Self {
+            virtual_ip,
+            virtual_netmask,
+            virtual_gateway,
+            virtual_network,
+            external_route,
+        }
+    }
+}
+
+#[cfg(target_os = "android")]
+impl Display for DeviceConfig {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.write_str(&format!(
+            "ip={} ,netmask={} ,gateway={}, external_route={:?}",
+            self.virtual_ip, self.virtual_netmask, self.virtual_gateway, self.external_route
+        ))
+    }
+}
+
+#[derive(Debug)]
+pub struct PeerClientInfo {
+    pub virtual_ip: Ipv4Addr,
+    pub name: String,
+    pub status: PeerDeviceStatus,
+    pub client_secret: bool,
+}
+
+impl PeerClientInfo {
+    pub fn new(
+        virtual_ip: Ipv4Addr,
+        name: String,
+        status: PeerDeviceStatus,
+        client_secret: bool,
+    ) -> Self {
+        Self {
+            virtual_ip,
+            name,
+            status,
+            client_secret,
+        }
+    }
+}
+
+impl Display for PeerClientInfo {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.write_str(&format!(
+            "ip={} ,name={} ,status={:?}, client_secret={}",
+            self.virtual_ip, self.name, self.status, self.client_secret
+        ))
+    }
+}
+
 pub trait VntCallback: Clone + Send + Sync + 'static {
    /// 启动成功
    fn success(&self) {}

    /// 创建网卡的信息
+    #[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))]
    fn create_tun(&self, _info: DeviceInfo) {}
    /// 连接
    fn connect(&self, _info: ConnectInfo) {}
@@ -199,6 +283,11 @@ pub trait VntCallback: Clone + Send + Sync + 'static {
    fn register(&self, _info: RegisterInfo) -> bool {
        true
    }
+    #[cfg(target_os = "android")]
+    fn generate_tun(&self, _info: DeviceConfig) -> u32 {
+        0
+    }
+    fn peer_client_list(&self, _info: Vec<PeerClientInfo>) {}
    /// 异常信息
    fn error(&self, _info: ErrorInfo) {}
    /// 服务停止
@@ -0,0 +1,216 @@
+use std::net::Ipv4Addr;
+use std::{io, process, thread};
+use std::sync::Arc;
+use std::sync::mpsc::{Receiver, SyncSender, TrySendError};
+use crossbeam_utils::atomic::AtomicCell;
+use tun::Device;
+use tun::device::IFace;
+use crate::handle::callback::DeviceConfig;
+use crate::protocol::NetPacket;
+use crate::util::{BufBlock, BufPool, GroupSyncSender, StopManager};
+
+pub struct TunAdapter {
+    #[cfg(any(target_os = "windows", target_os = "linux"))] is_tap: bool,
+    #[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))] device_name: Option<String>,
+    #[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))] mtu: u32,
+    #[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))] route_record: Vec<(Ipv4Addr, Ipv4Addr)>,
+    device: Option<Arc<Device>>,
+    buf_pool: BufPool,
+    stop_manager: StopManager,
+    receiver_stage: Option<Receiver<NetPacket<BufBlock>>>,
+    sender_stage: Option<SyncSender<NetPacket<BufBlock>>>,
+}
+impl TunAdapter {
+    pub fn new(
+        #[cfg(any(target_os = "windows", target_os = "linux"))] is_tap: bool,
+        #[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))] device_name: Option<String>,
+        #[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))] mtu: u32,
+        #[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))] route_record: Vec<(Ipv4Addr, Ipv4Addr)>,
+        buf_pool: BufPool,
+        stop_manager: StopManager,
+        receiver: Receiver<NetPacket<BufBlock>>,
+        sender: SyncSender<NetPacket<BufBlock>>,
+    )->Self{
+        Self{
+            #[cfg(any(target_os = "windows", target_os = "linux"))] is_tap,
+            #[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))]  device_name,
+            #[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))]  mtu,
+            #[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))]   route_record,
+            device: None,
+            buf_pool,
+            stop_manager,
+            receiver_stage: Some(receiver),
+            sender_stage: Some(sender),
+        }
+    }
+}
+
+impl TunAdapter {
+    pub fn device(&mut self, #[cfg(target_os = "android")] device_fd: u32) -> io::Result<()> {
+        if self.device.is_some() {
+            return Ok(());
+        } else {
+            let device = create_device(#[cfg(any(target_os = "windows", target_os = "linux"))] self.is_tap,
+                                       #[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))] self.device_name.clone(),
+                                       #[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))] self.mtu,
+                                       #[cfg(target_os = "android")] device_fd, )?;
+            let device = Arc::new(device);
+            self.device.replace(device);
+            {
+                let device = device.clone();
+                let buf_pool = self.buf_pool.clone();
+                let stop_manager = self.stop_manager.clone();
+                let sender = self.sender_stage.take().unwrap();
+                thread::Builder::new().name("tun-read".into()).spawn(move || {
+                    loop {
+                        let mut buf_block = buf_pool.alloc();
+                        #[cfg(not(target_os = "macos"))]
+                            let start = 12;
+                        #[cfg(target_os = "macos")]
+                            let start = 8;
+                        match device.read(&mut buf_block.as_mut()[start..]) {
+                            Ok(len) => {
+                                buf_block.as_mut()[..12].fill(0);
+                                buf_block.set_data_len(start + len);
+                                if let Err(e) = sender.try_send(buf_block) {
+                                    match e {
+                                        TrySendError::Full(_) => {
+                                            log::warn!("发生丢包");
+                                        }
+                                        TrySendError::Disconnected(_) => {
+                                            break;
+                                        }
+                                    }
+                                }
+                            }
+                            Err(e) => {
+                                log::warn!("{:?}",e);
+                                break;
+                            }
+                        }
+                    }
+                    stop_manager.stop();
+                }).unwrap();
+            }
+            {
+                let device = device.clone();
+                let stop_manager = self.stop_manager.clone();
+                let receiver = self.receiver_stage.take().unwrap();
+
+                thread::Builder::new().name("tun-write".into()).spawn(move || {
+                    while let Ok(data) = receiver.recv() {
+                        if let Err(e) = device.write(data.as_data()) {
+                            log::warn!("写入网卡失败:{}",e);
+                            break;
+                        }
+                    }
+                    stop_manager.stop();
+                }).unwrap();
+            }
+        }
+    }
+    #[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))]
+    pub fn change_ip(&mut self, info: DeviceConfig) -> io::Result<()> {
+        let device = if let Some(device) = &self.device {
+            device
+        } else {
+            return Err(io::Error::new(io::ErrorKind::NotFound, "IFace"));
+        };
+        device.set_ip(info.virtual_ip, info.virtual_netmask)?;
+        for (dest, mask) in self.route_record.drain(..) {
+            if let Err(e) = self.device.delete_route(dest, mask) {
+                log::warn!("删除路由失败 ={:?}", e);
+            }
+        }
+        if let Err(e) = device.add_route(info.virtual_network, info.virtual_netmask, 1)
+        {
+            log::warn!("添加默认路由失败 ={:?}", e);
+        } else {
+            self.route_record.push((info.virtual_network, info.virtual_netmask));
+        }
+        if let Err(e) = device
+            .add_route(Ipv4Addr::BROADCAST, Ipv4Addr::BROADCAST, 1)
+        {
+            log::warn!("添加广播路由失败 ={:?}", e);
+        } else {
+            self.route_record.push((Ipv4Addr::BROADCAST, Ipv4Addr::BROADCAST));
+        }
+
+        if let Err(e) = device.add_route(
+            Ipv4Addr::from([224, 0, 0, 0]),
+            Ipv4Addr::from([240, 0, 0, 0]),
+            1,
+        ) {
+            log::warn!("添加组播路由失败 ={:?}", e);
+        } else {
+            self.route_record.push((
+                Ipv4Addr::from([224, 0, 0, 0]),
+                Ipv4Addr::from([240, 0, 0, 0]),
+            ));
+        }
+
+        for (dest, mask) in info.external_route {
+            if let Err(e) = self.device.add_route(dest, mask, 1) {
+                log::warn!("添加路由失败 ={:?}", e);
+            } else {
+                self.route_record.push((dest, mask));
+            }
+        }
+        Ok(())
+    }
+}
+
+#[cfg(any(target_os = "windows", target_os = "linux"))]
+const DEFAULT_TUN_NAME: &str = "vnt-tun";
+#[cfg(any(target_os = "windows", target_os = "linux"))]
+const DEFAULT_TAP_NAME: &str = "vnt-tap";
+
+pub fn create_device(#[cfg(any(target_os = "windows", target_os = "linux"))] is_tap: bool,
+                     #[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))] device_name: Option<String>,
+                     #[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))] mtu: u32,
+                     #[cfg(target_os = "android")] device_fd: u32, ) -> io::Result<Device> {
+    #[cfg(any(target_os = "windows", target_os = "linux"))]
+        let default_name: &str = if is_tap {
+        DEFAULT_TAP_NAME
+    } else {
+        DEFAULT_TUN_NAME
+    };
+    #[cfg(target_os = "linux")]
+        let device = {
+        let device_name = device_name
+            .unwrap_or(default_name.to_string());
+        if &device_name == default_name {
+            delete_device(default_name);
+        }
+        Device::new(Some(device_name), is_tap)?
+    };
+    #[cfg(target_os = "macos")]
+        let device = Device::new(device_name)?;
+    #[cfg(target_os = "windows")]
+        let device = Device::new(
+        device_name
+            .unwrap_or(default_name.to_string()),
+        is_tap,
+    )?;
+    #[cfg(target_os = "android")]
+        let device = Device::new(device_fd as _)?;
+    #[cfg(not(target_os = "android"))]
+    device.set_mtu(mtu)?;
+    Ok(device)
+}
+
+#[cfg(target_os = "linux")]
+fn delete_device(name: &str) {
+    // 删除默认网卡，此操作有风险，后续可能去除
+    use std::process::Command;
+    let cmd = format!("ip link delete {}", name);
+    let delete_tun = Command::new("sh")
+        .arg("-c")
+        .arg(&cmd)
+        .output()
+        .expect("sh exec error!");
+    if !delete_tun.status.success() {
+        log::info!("{},{:?}",cmd, delete_tun);
+    }
+}
+
@@ -0,0 +1,242 @@
+use std::{io, thread};
+use std::net::Ipv4Addr;
+use std::sync::Arc;
+use std::sync::mpsc::Receiver;
+
+use crossbeam_utils::atomic::AtomicCell;
+use parking_lot::Mutex;
+
+use packet::icmp::icmp::IcmpPacket;
+use packet::icmp::Kind;
+use packet::ip::ipv4;
+use packet::ip::ipv4::packet::IpV4Packet;
+use packet::ip::ipv4::protocol::Protocol;
+
+use crate::channel::context::Context;
+use crate::cipher::Cipher;
+use crate::external_route::ExternalRoute;
+use crate::handle::{check_dest, CurrentDeviceInfo, PeerDeviceInfo};
+use crate::ip_proxy::{IpProxyMap, ProxyHandler};
+use crate::protocol;
+use crate::protocol::{ip_turn_packet, MAX_TTL, NetPacket, Version};
+use crate::protocol::body::ENCRYPTION_RESERVED;
+use crate::protocol::ip_turn_packet::BroadcastPacket;
+use crate::util::{BufBlock, SingleU64Adder, StopManager};
+pub mod adapter;
+pub fn start(
+    receivers: Vec<Receiver<BufBlock>>,
+    context: Context,
+    current_device: Arc<AtomicCell<CurrentDeviceInfo>>,
+    ip_route: ExternalRoute,
+    #[cfg(feature = "ip_proxy")] ip_proxy_map: Option<IpProxyMap>,
+    client_cipher: Cipher,
+    server_cipher: Cipher,
+    device_list: Arc<Mutex<(u16, Vec<PeerDeviceInfo>)>>,
+) -> io::Result<()> {
+    for (index,receiver) in receivers.into_iter().enumerate() {
+        let context = context.clone();
+        let current_device = current_device.clone();
+        let ip_route = ip_route.clone();
+        #[cfg(feature = "ip_proxy")]
+            let ip_proxy_map = ip_proxy_map.clone();
+        let client_cipher = client_cipher.clone();
+        let server_cipher = server_cipher.clone();
+        let device_list = device_list.clone();
+        thread::Builder::new()
+            .name(format!("IFace-{}", index))
+            .spawn(move || {
+                while let Ok(mut data) = receiver.recv() {
+                    let data_len = data.data_len();
+                    let buf = data.as_start_mut();
+                    if data_len==0{
+                        break;
+                    }
+                    match handle(
+                        &context,
+                        buf,
+                        data_len,
+                        current_device.load(),
+                        &ip_route,
+                        #[cfg(feature = "ip_proxy")]
+                            &ip_proxy_map,
+                        &client_cipher,
+                        &server_cipher,
+                        &device_list,
+                    ) {
+                        Ok(_) => {}
+                        Err(e) => {
+                            log::warn!("{:?}", e)
+                        }
+                    }
+                }
+            })?;
+    }
+    Ok(())
+}
+
+
+/// 实现一个原地发送，必须保证是如下结构
+/// |12字节开头|ip报文|至少1024字节结尾|
+///
+pub fn handle(
+    context: &Context,
+    buf: &mut [u8],
+    data_len: usize, //数据总长度=12+ip包长度
+    current_device: CurrentDeviceInfo,
+    ip_route: &ExternalRoute,
+    #[cfg(feature = "ip_proxy")] proxy_map: &Option<IpProxyMap>,
+    client_cipher: &Cipher,
+    server_cipher: &Cipher,
+    device_list: &Mutex<(u16, Vec<PeerDeviceInfo>)>,
+) -> io::Result<()> {
+    let ipv4_packet = IpV4Packet::new(&buf[12..data_len])?;
+    let protocol = ipv4_packet.protocol();
+    let src_ip = ipv4_packet.source_ip();
+    let mut dest_ip = ipv4_packet.destination_ip();
+    let mut net_packet = NetPacket::new0(data_len, buf)?;
+    net_packet.set_version(Version::V1);
+    net_packet.set_protocol(protocol::Protocol::IpTurn);
+    net_packet.set_transport_protocol(ip_turn_packet::Protocol::Ipv4.into());
+    net_packet.first_set_ttl(6);
+    net_packet.set_source(src_ip);
+    net_packet.set_destination(dest_ip);
+    if dest_ip == current_device.virtual_gateway {
+        // 发到网关的加密方式不一样，要单独处理
+        if protocol == Protocol::Icmp {
+            net_packet.set_gateway_flag(true);
+            server_cipher.encrypt_ipv4(&mut net_packet)?;
+            context.send_default(net_packet.buffer(), current_device.connect_server)?;
+        }
+        return Ok(());
+    }
+    if dest_ip.is_multicast() {
+        //当作广播处理
+        dest_ip = Ipv4Addr::BROADCAST;
+        net_packet.set_destination(Ipv4Addr::BROADCAST);
+    }
+    if dest_ip.is_broadcast() || current_device.broadcast_ip == dest_ip {
+        // 广播 发送到直连目标
+        client_cipher.encrypt_ipv4(&mut net_packet)?;
+        broadcast(
+            server_cipher,
+            context,
+            &mut net_packet,
+            &current_device,
+            device_list,
+        )?;
+        return Ok(());
+    }
+    if !check_dest(
+        dest_ip,
+        current_device.virtual_netmask,
+        current_device.virtual_network,
+    ) {
+        if let Some(r_dest_ip) = ip_route.route(&dest_ip) {
+            //路由的目标不能是自己
+            if r_dest_ip == src_ip {
+                return Ok(());
+            }
+            //需要修改目的地址
+            dest_ip = r_dest_ip;
+            net_packet.set_destination(r_dest_ip);
+        } else {
+            return Ok(());
+        }
+    }
+    #[cfg(feature = "ip_proxy")]
+    if let Some(proxy_map) = proxy_map {
+        let mut ipv4_packet = IpV4Packet::new(net_packet.payload_mut())?;
+        proxy_map.send_handle(&mut ipv4_packet)?;
+    }
+    client_cipher.encrypt_ipv4(&mut net_packet)?;
+    context.send_ipv4_by_id(
+        net_packet.buffer(),
+        &dest_ip,
+        current_device.connect_server,
+        current_device.status.online(),
+    )
+}
+
+fn broadcast(
+    server_cipher: &Cipher,
+    sender: &Context,
+    net_packet: &mut NetPacket<&mut [u8]>,
+    current_device: &CurrentDeviceInfo,
+    device_list: &Mutex<(u16, Vec<PeerDeviceInfo>)>,
+) -> io::Result<()> {
+    let list: Vec<Ipv4Addr> = device_list
+        .lock()
+        .1
+        .iter()
+        .filter(|info| info.status.is_online())
+        .map(|info| info.virtual_ip)
+        .collect();
+    const MAX_COUNT: usize = 8;
+    let mut p2p_ips = Vec::with_capacity(8);
+    let mut relay_ips = Vec::with_capacity(8);
+    let mut overflow = false;
+    for (index, peer_ip) in list.into_iter().enumerate() {
+        if index > MAX_COUNT {
+            overflow = true;
+            break;
+        }
+        if let Some(route) = sender.route_table.route_one_p2p(&peer_ip) {
+            if sender
+                .send_by_key(net_packet.buffer(), route.route_key())
+                .is_ok()
+            {
+                p2p_ips.push(peer_ip);
+                continue;
+            }
+        }
+        relay_ips.push(peer_ip);
+    }
+    if !overflow && relay_ips.is_empty() {
+        //全部p2p,不需要服务器中转
+        return Ok(());
+    }
+
+    if p2p_ips.is_empty() {
+        //都没有p2p则直接由服务器转发
+        if current_device.status.online() {
+            sender.send_default(net_packet.buffer(), current_device.connect_server)?;
+        }
+        return Ok(());
+    }
+    if !overflow && relay_ips.len() == 2 {
+        // 如果转发的ip数不多就直接发
+        for peer_ip in relay_ips {
+            //非直连的广播要改变目的地址，不然服务端收到了会再次广播
+            net_packet.set_destination(peer_ip);
+            sender.send_ipv4_by_id(
+                net_packet.buffer(),
+                &peer_ip,
+                current_device.connect_server,
+                current_device.status.online(),
+            )?;
+        }
+        return Ok(());
+    }
+    if current_device.status.offline() {
+        //离线的不再转发
+        return Ok(());
+    }
+    let buf = vec![0u8; 12 + 1 + p2p_ips.len() * 4 + net_packet.data_len() + ENCRYPTION_RESERVED];
+    //剩余的发送到服务端，需要告知哪些已发送过
+    let mut server_packet = NetPacket::new_encrypt(buf)?;
+    server_packet.set_version(Version::V1);
+    server_packet.set_gateway_flag(true);
+    server_packet.first_set_ttl(MAX_TTL);
+    server_packet.set_source(net_packet.source());
+    //使用对应的目的地址
+    server_packet.set_destination(net_packet.destination());
+    server_packet.set_protocol(protocol::Protocol::IpTurn);
+    server_packet.set_transport_protocol(ip_turn_packet::Protocol::Ipv4Broadcast.into());
+
+    let mut broadcast = BroadcastPacket::unchecked(server_packet.payload_mut());
+    broadcast.set_address(&p2p_ips)?;
+    broadcast.set_data(net_packet.buffer())?;
+    server_cipher.encrypt_ipv4(&mut server_packet)?;
+    sender.send_default(server_packet.buffer(), current_device.connect_server)
+}
+
@@ -1,16 +1,13 @@
+use parking_lot::RwLock;
+use protobuf::Message;
 use std::collections::HashMap;
 use std::io;
 use std::net::{Ipv4Addr, Ipv6Addr};
 use std::sync::Arc;

-use parking_lot::RwLock;
-use protobuf::Message;
-
 use packet::icmp::{icmp, Kind};
 use packet::ip::ipv4;
 use packet::ip::ipv4::packet::IpV4Packet;
-use tun::device::IFace;
-use tun::Device;

 use crate::channel::context::Context;
 use crate::channel::punch::NatInfo;
@@ -29,11 +26,13 @@ use crate::protocol::control_packet::ControlPacket;
 use crate::protocol::{
    control_packet, ip_turn_packet, other_turn_packet, NetPacket, Protocol, Version, MAX_TTL,
 };
-
+use crate::tun_tap_device::tun_create_helper::DeviceAdapter;
+#[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))]
+use tun::device::IFace;
 /// 处理来源于客户端的包
 #[derive(Clone)]
 pub struct ClientPacketHandler {
-    device: Arc<Device>,
+    device: DeviceAdapter,
    client_cipher: Cipher,
    punch_sender: PunchSender,
    peer_nat_info_map: Arc<RwLock<HashMap<Ipv4Addr, NatInfo>>>,
@@ -45,7 +44,7 @@ pub struct ClientPacketHandler {

 impl ClientPacketHandler {
    pub fn new(
-        device: Arc<Device>,
+        device: DeviceAdapter,
        client_cipher: Cipher,
        punch_sender: PunchSender,
        peer_nat_info_map: Arc<RwLock<HashMap<Ipv4Addr, NatInfo>>>,
@@ -6,8 +6,6 @@ use std::{io, thread};
 use crossbeam_utils::atomic::AtomicCell;
 use parking_lot::{Mutex, RwLock};

-use tun::Device;
-
 use crate::channel::context::Context;
 use crate::channel::handler::RecvChannelHandler;
 use crate::channel::punch::NatInfo;
@@ -27,6 +25,7 @@ use crate::handle::{BaseConfigInfo, CurrentDeviceInfo, PeerDeviceInfo, SELF_IP};
 use crate::ip_proxy::IpProxyMap;
 use crate::nat::NatTest;
 use crate::protocol::NetPacket;
+use crate::tun_tap_device::tun_create_helper::DeviceAdapter;
 use crate::util::U64Adder;

 mod client;
@@ -56,7 +55,7 @@ impl<Call: VntCallback> RecvDataHandler<Call> {
        server_cipher: Cipher,
        client_cipher: Cipher,
        current_device: Arc<AtomicCell<CurrentDeviceInfo>>,
-        device: Arc<Device>,
+        device: DeviceAdapter,
        device_list: Arc<Mutex<(u16, Vec<PeerDeviceInfo>)>>,
        config_info: BaseConfigInfo,
        nat_test: NatTest,
@@ -11,8 +11,6 @@ use protobuf::Message;
 use packet::icmp::{icmp, Kind};
 use packet::ip::ipv4;
 use packet::ip::ipv4::packet::IpV4Packet;
-use tun::device::IFace;
-use tun::Device;

 use crate::channel::context::Context;
 use crate::channel::{Route, RouteKey};
@@ -29,12 +27,15 @@ use crate::handle::{
    registrar, BaseConfigInfo, ConnectStatus, CurrentDeviceInfo, PeerDeviceInfo, GATEWAY_IP,
 };
 use crate::nat::NatTest;
-use crate::proto;
 use crate::proto::message::{DeviceList, HandshakeResponse, RegistrationResponse};
 use crate::protocol::body::ENCRYPTION_RESERVED;
 use crate::protocol::control_packet::ControlPacket;
 use crate::protocol::error_packet::InErrorPacket;
 use crate::protocol::{ip_turn_packet, service_packet, NetPacket, Protocol, Version, MAX_TTL};
+use crate::tun_tap_device::tun_create_helper::DeviceAdapter;
+use crate::{proto, PeerClientInfo};
+#[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))]
+use tun::device::IFace;

 /// 处理来源于服务端的包
 #[derive(Clone)]
@@ -43,13 +44,14 @@ pub struct ServerPacketHandler<Call> {
    rsa_cipher: Arc<Mutex<Option<RsaCipher>>>,
    server_cipher: Cipher,
    current_device: Arc<AtomicCell<CurrentDeviceInfo>>,
-    device: Arc<Device>,
+    device: DeviceAdapter,
    device_list: Arc<Mutex<(u16, Vec<PeerDeviceInfo>)>>,
    config_info: BaseConfigInfo,
    nat_test: NatTest,
    callback: Call,
    #[cfg(feature = "server_encrypt")]
    up_key_time: Arc<AtomicCell<Instant>>,
+    #[cfg(not(target_os = "android"))]
    route_record: Arc<Mutex<Vec<(Ipv4Addr, Ipv4Addr)>>>,
    external_route: ExternalRoute,
    handshake: Handshake,
@@ -60,7 +62,7 @@ impl<Call> ServerPacketHandler<Call> {
        #[cfg(feature = "server_encrypt")] rsa_cipher: Arc<Mutex<Option<RsaCipher>>>,
        server_cipher: Cipher,
        current_device: Arc<AtomicCell<CurrentDeviceInfo>>,
-        device: Arc<Device>,
+        device: DeviceAdapter,
        device_list: Arc<Mutex<(u16, Vec<PeerDeviceInfo>)>>,
        config_info: BaseConfigInfo,
        nat_test: NatTest,
@@ -80,6 +82,7 @@ impl<Call> ServerPacketHandler<Call> {
            callback,
            #[cfg(feature = "server_encrypt")]
            up_key_time: Arc::new(AtomicCell::new(Instant::now() - Duration::from_secs(60))),
+            #[cfg(not(target_os = "android"))]
            route_record: Arc::new(Mutex::default()),
            external_route,
            handshake,
@@ -299,6 +302,31 @@ impl<Call: VntCallback> ServerPacketHandler<Call> {
                        if old.virtual_ip != Ipv4Addr::UNSPECIFIED {
                            log::info!("ip发生变化,old:{:?},response={:?}", old, response);
                        }
+                        #[cfg(target_os = "android")]
+                        {
+                            let device_config = crate::handle::callback::DeviceConfig::new(
+                                virtual_ip,
+                                virtual_netmask,
+                                virtual_gateway,
+                                virtual_network,
+                                self.external_route.to_route(),
+                            );
+                            let device_fd = self.callback.generate_tun(device_config);
+                            if device_fd == 0 {
+                                self.callback.error(ErrorInfo::new_msg(
+                                    ErrorType::Unknown,
+                                    "device_fd == 0".into(),
+                                ));
+                            } else {
+                                let device = Arc::new(tun::Device::new(device_fd as _)?);
+                                if let Err(e) = self.device.start(device) {
+                                    self.callback.error(ErrorInfo::new_msg(
+                                        ErrorType::Unknown,
+                                        format!("{:?}", e),
+                                    ));
+                                }
+                            }
+                        }
                        #[cfg(not(target_os = "android"))]
                        {
                            if let Err(e) = self.device.set_ip(virtual_ip, virtual_netmask) {
@@ -393,10 +421,18 @@ impl<Call: VntCallback> ServerPacketHandler<Call> {
                )
            })
            .collect();
-        let mut dev = self.device_list.lock();
-        //这里可能会收到旧的消息，但是随着时间推移总会收到新的
-        dev.0 = epoch;
-        dev.1 = ip_list;
+        {
+            let mut dev = self.device_list.lock();
+            //这里可能会收到旧的消息，但是随着时间推移总会收到新的
+            dev.0 = epoch;
+            dev.1 = ip_list.clone();
+        }
+        self.callback.peer_client_list(
+            ip_list
+                .into_iter()
+                .map(|v| PeerClientInfo::new(v.virtual_ip, v.name, v.status, v.client_secret))
+                .collect(),
+        );
    }
    fn register(&self, current_device: &CurrentDeviceInfo, context: &Context) -> io::Result<()> {
        if current_device.status.online() {
@@ -87,14 +87,14 @@ pub fn start(
    device_list: Arc<Mutex<(u16, Vec<PeerDeviceInfo>)>>,
 ) -> io::Result<()> {
    let worker = {
-        #[cfg(target_os = "macos")]
+        #[cfg(any(target_os = "macos", target_os = "android"))]
        let current_device = current_device.clone();
        let device = device.clone();
        stop_manager.add_listener("tun_device".into(), move || {
            if let Err(e) = device.shutdown() {
                log::warn!("{:?}", e);
            }
-            #[cfg(target_os = "macos")]
+            #[cfg(any(target_os = "macos", target_os = "android"))]
            {
                let ip = current_device.load().virtual_ip;
                if let Ok(udp) = std::net::UdpSocket::bind("0.0.0.0:0") {
@@ -13,4 +13,4 @@ pub mod protocol;
 pub mod tun_tap_device;
 pub mod util;

-pub use handle::callback::{DeviceInfo, ErrorInfo, HandshakeInfo, RegisterInfo, VntCallback};
+pub use handle::callback::*;
@@ -0,0 +1,64 @@
+use std::io;
+use std::sync::Arc;
+use tun::device::IFace;
+use tun::Device;
+
+#[cfg(any(target_os = "windows", target_os = "linux"))]
+const DEFAULT_TUN_NAME: &str = "vnt-tun";
+#[cfg(any(target_os = "windows", target_os = "linux"))]
+const DEFAULT_TAP_NAME: &str = "vnt-tap";
+
+#[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))]
+pub fn create_device(config: &crate::core::Config) -> io::Result<Arc<Device>> {
+    #[cfg(any(target_os = "windows", target_os = "linux"))]
+    let default_name: &str = if config.tap {
+        DEFAULT_TAP_NAME
+    } else {
+        DEFAULT_TUN_NAME
+    };
+    #[cfg(target_os = "linux")]
+    let device = {
+        let device_name = config
+            .device_name
+            .clone()
+            .unwrap_or(default_name.to_string());
+        if &device_name == default_name {
+            delete_device(default_name);
+        }
+        Arc::new(Device::new(Some(device_name), config.tap)?)
+    };
+    #[cfg(target_os = "macos")]
+    let device = Arc::new(Device::new(config.device_name.clone())?);
+    #[cfg(target_os = "windows")]
+    let device = Arc::new(Device::new(
+        config
+            .device_name
+            .clone()
+            .unwrap_or(default_name.to_string()),
+        config.tap,
+    )?);
+    let mtu = config.mtu.unwrap_or_else(|| {
+        if config.password.is_none() {
+            1450
+        } else {
+            1410
+        }
+    });
+    device.set_mtu(mtu)?;
+    Ok(device)
+}
+
+#[cfg(target_os = "linux")]
+fn delete_device(name: &str) {
+    // 删除默认网卡，此操作有风险，后续可能去除
+    use std::process::Command;
+    let cmd = format!("ip link delete {}", name);
+    let delete_tun = Command::new("sh")
+        .arg("-c")
+        .arg(&cmd)
+        .output()
+        .expect("sh exec error!");
+    if !delete_tun.status.success() {
+        log::warn!("删除网卡失败:{:?}", delete_tun);
+    }
+}
@@ -1,70 +1,6 @@
-use std::io;
-use std::sync::Arc;
+#[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))]
+pub use create_device::create_device;

-use tun::device::IFace;
-use tun::Device;
-
-use crate::core::Config;
-#[cfg(any(target_os = "windows", target_os = "linux"))]
-const DEFAULT_TUN_NAME: &str = "vnt-tun";
-#[cfg(any(target_os = "windows", target_os = "linux"))]
-const DEFAULT_TAP_NAME: &str = "vnt-tap";
-
-pub fn create_device(config: &Config) -> io::Result<Arc<Device>> {
-    #[cfg(any(target_os = "windows", target_os = "linux"))]
-    let default_name: &str = if config.tap {
-        DEFAULT_TAP_NAME
-    } else {
-        DEFAULT_TUN_NAME
-    };
-    #[cfg(target_os = "linux")]
-    let device = {
-        let device_name = config
-            .device_name
-            .clone()
-            .unwrap_or(default_name.to_string());
-        if &device_name == default_name {
-            delete_device(default_name);
-        }
-        Arc::new(Device::new(Some(device_name), config.tap)?)
-    };
-    #[cfg(target_os = "macos")]
-    let device = Arc::new(Device::new(config.device_name.clone())?);
-    #[cfg(target_os = "windows")]
-    let device = Arc::new(Device::new(
-        config
-            .device_name
-            .clone()
-            .unwrap_or(default_name.to_string()),
-        config.tap,
-    )?);
-    #[cfg(target_os = "android")]
-    let device = Arc::new(Device::new(config.device_fd as _)?);
-    #[cfg(not(target_os = "android"))]
-    {
-        let mtu = config.mtu.unwrap_or_else(|| {
-            if config.password.is_none() {
-                1450
-            } else {
-                1410
-            }
-        });
-        device.set_mtu(mtu)?;
-    }
-    Ok(device)
-}
-
-#[cfg(target_os = "linux")]
-fn delete_device(name: &str) {
-    // 删除默认网卡，此操作有风险，后续可能去除
-    use std::process::Command;
-    let cmd = format!("ip link delete {}", name);
-    let delete_tun = Command::new("sh")
-        .arg("-c")
-        .arg(&cmd)
-        .output()
-        .expect("sh exec error!");
-    if !delete_tun.status.success() {
-        log::warn!("删除网卡失败:{:?}", delete_tun);
-    }
-}
+#[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))]
+mod create_device;
+pub mod tun_create_helper;
@@ -0,0 +1,134 @@
+use std::io;
+use std::sync::Arc;
+
+use crossbeam_utils::atomic::AtomicCell;
+use parking_lot::Mutex;
+
+use tun::Device;
+
+use crate::channel::context::Context;
+use crate::cipher::Cipher;
+use crate::external_route::ExternalRoute;
+use crate::handle::{CurrentDeviceInfo, PeerDeviceInfo};
+use crate::ip_proxy::IpProxyMap;
+use crate::util::{SingleU64Adder, StopManager};
+#[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))]
+#[repr(transparent)]
+#[derive(Clone)]
+pub struct DeviceAdapter {
+    tun: Arc<Device>,
+}
+impl DeviceAdapter {
+    #[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))]
+    pub fn new(tun: Arc<Device>) -> Self {
+        Self { tun }
+    }
+    #[cfg(target_os = "android")]
+    pub fn new(tun_device_helper: TunDeviceHelper) -> Self {
+        Self {
+            tun: Arc::new(Mutex::new(None)),
+            tun_device_helper,
+        }
+    }
+}
+#[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))]
+impl std::ops::Deref for DeviceAdapter {
+    type Target = Arc<Device>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.tun
+    }
+}
+
+#[cfg(target_os = "android")]
+#[derive(Clone)]
+pub struct DeviceAdapter {
+    tun: Arc<Mutex<Option<Arc<Device>>>>,
+    tun_device_helper: TunDeviceHelper,
+}
+#[cfg(target_os = "android")]
+impl DeviceAdapter {
+    pub fn write(&self, buf: &[u8]) -> io::Result<usize> {
+        if let Some(device) = self.tun.lock().as_ref() {
+            use tun::device::IFace;
+            device.write(buf)
+        } else {
+            Err(io::Error::new(io::ErrorKind::Other, "not tun device"))
+        }
+    }
+    pub fn start(&self, device: Arc<Device>) -> io::Result<()> {
+        self.tun_device_helper.start(device.clone())?;
+        self.tun.lock().replace(device);
+        Ok(())
+    }
+}
+
+#[derive(Clone)]
+pub struct TunDeviceHelper {
+    inner: Arc<AtomicCell<Option<TunDeviceHelperInner>>>,
+}
+
+struct TunDeviceHelperInner {
+    stop_manager: StopManager,
+    context: Context,
+    current_device: Arc<AtomicCell<CurrentDeviceInfo>>,
+    ip_route: ExternalRoute,
+    #[cfg(feature = "ip_proxy")]
+    ip_proxy_map: Option<IpProxyMap>,
+    client_cipher: Cipher,
+    server_cipher: Cipher,
+    parallel: usize,
+    up_counter: SingleU64Adder,
+    device_list: Arc<Mutex<(u16, Vec<PeerDeviceInfo>)>>,
+}
+
+impl TunDeviceHelper {
+    pub fn new(
+        stop_manager: StopManager,
+        context: Context,
+        current_device: Arc<AtomicCell<CurrentDeviceInfo>>,
+        ip_route: ExternalRoute,
+        #[cfg(feature = "ip_proxy")] ip_proxy_map: Option<IpProxyMap>,
+        client_cipher: Cipher,
+        server_cipher: Cipher,
+        parallel: usize,
+        up_counter: SingleU64Adder,
+        device_list: Arc<Mutex<(u16, Vec<PeerDeviceInfo>)>>,
+    ) -> Self {
+        Self {
+            inner: Arc::new(AtomicCell::new(Some(TunDeviceHelperInner {
+                stop_manager,
+                context,
+                current_device,
+                ip_route,
+                ip_proxy_map,
+                client_cipher,
+                server_cipher,
+                parallel,
+                up_counter,
+                device_list,
+            }))),
+        }
+    }
+    pub fn start(&self, device: Arc<Device>) -> io::Result<()> {
+        if let Some(inner) = self.inner.take() {
+            crate::handle::tun_tap::tun_handler::start(
+                inner.stop_manager,
+                inner.context,
+                device,
+                inner.current_device,
+                inner.ip_route,
+                #[cfg(feature = "ip_proxy")]
+                inner.ip_proxy_map,
+                inner.client_cipher,
+                inner.server_cipher,
+                inner.parallel,
+                inner.up_counter,
+                inner.device_list,
+            )?;
+            Ok(())
+        } else {
+            Err(io::Error::new(io::ErrorKind::Other, "Repeated start"))
+        }
+    }
+}