jinleileiking.github.com

View on GitHub

go

go build, go get 都不行,是因为目录名和go原始包里的名字冲突了:sync

[root@vm10-65-136-13 sync]# go build consumer.go
panic: runtime error: slice bounds out of range

goroutine 1 [running]:
panic(0x7e1880, 0xc420012110)
        /usr/local/go/src/runtime/panic.go:500 +0x1a1
main.vendoredImportPath(0xc4203bc900, 0xc4203e88a1, 0x1f, 0x3, 0x0)
        /usr/local/go/src/cmd/go/pkg.go:463 +0x66c
main.loadImport(0xc4203e88a1, 0x1f, 0xc420250a80, 0x34, 0xc4203bc900, 0xc4201cb7e0, 0xc4203e8a20, 0x1, 0x1, 0x1, ...)
        /usr/local/go/src/cmd/go/pkg.go:333 +0x9ea
main.(*Package).load(0xc4203bc900, 0xc4201cb7e0, 0xc4204ae700, 0x0, 0x0, 0x4)
        /usr/local/go/src/cmd/go/pkg.go:940 +0x12b6
main.loadImport(0xc42019c3e1, 0x19, 0xc42019c060, 0x13, 0xc4201a6000, 0xc4201cb7e0, 0xc4201a24e0, 0x1, 0x1, 0x1, ...)
        /usr/local/go/src/cmd/go/pkg.go:374 +0x470
main.(*Package).load(0xc4201a6000, 0xc4201cb7e0, 0xc42017bc00, 0x0, 0x0, 0xc4201986a0)
        /usr/local/go/src/cmd/go/pkg.go:940 +0x12b6
main.goFilesPackage(0xc42000c260, 0x1, 0x1, 0x3)
        /usr/local/go/src/cmd/go/build.go:830 +0x87e
main.packagesAndErrors(0xc42000c260, 0x1, 0x1, 0xc420198320, 0x20, 0x7f3a00)
        /usr/local/go/src/cmd/go/pkg.go:1687 +0x4f7
main.packagesForBuild(0xc42000c260, 0x1, 0x1, 0x893cc8, 0xc420054000, 0x0)
        /usr/local/go/src/cmd/go/pkg.go:1719 +0x75
main.runBuild(0xa27400, 0xc42000c260, 0x1, 0x1)
        /usr/local/go/src/cmd/go/build.go:440 +0xd4
main.main()
        /usr/local/go/src/cmd/go/main.go:181 +0x624`

redis

导入多个rdb给一个redis

rdb --c protocol ~/oldata/shipinyun_redisdump/dump_5.rdb| ./redis-cli --pipe

pipe是高版本支持的

superset

python3.5 安装

添加vvv, 监控缓慢的安装过程,用绝对目录

CFLAGS="-I/home/jinlei1/tools/openssl-1.1.0/openssl/include" LDFLAGS="-L/home/jinlei1/tools/openssl-1.1.0/openssl/lib" pip3.5 install --no-use-wheel superset -vvvv

启动

superset runserver -p 8088

nginx rtmp

nginx 启动

ngx_rtmp.c 定义 ngx_rtmp_module, ngx_module_t 结构体

ngx_rtmp_commands 处理nginx.conf里的配置项,调用ngx_rtmp_block。

ngx_rtmp_block
  ngx_rtmp_optimize_servers
    ngx_rtmp_init_listening
      ngx_rtmp_add_listening
        ngx_create_listening
        ls->handler = ngx_rtmp_init_connection

handshake

c0+c1   –> client一起发送

s0+s1+s2 <— server分次多次回调发送

c2 —>

ngx_rtmp_init_connection
  ngx_rtmp_init_session
    ngx_rtmp_set_chunk_size   
  ngx_rtmp_handshake
    ngx_rtmp_handshake_recv
      ngx_rtmp_handshake_parse_challenge
      case NGX_RTMP_HANDSHAKE_SERVER_SEND_CHALLENGE
      ngx_rtmp_handshake_send
        case NGX_RTMP_HANDSHAKE_SERVER_SEND_RESPONSE:
        ngx_rtmp_handshake_send
          case NGX_RTMP_HANDSHAKE_SERVER_RECV_RESPONSE:
            ngx_rtmp_handshake_recv
           .....
           case NGX_RTMP_HANDSHAKE_SERVER_DONE:
            ngx_rtmp_handshake_done
              ngx_rtmp_cycle !!!!!!!!!!!!
                ngx_rtmp_reset_ping
                  ngx_add_timer
                  
           

connect

ngx_rtmp_handshake_done调用

ngx_rtmp_cycle
  ngx_rtmp_recv

to solve…..

-   1011 static ngx_rtmp_amf_handler_t ngx_rtmp_cmd_map[] = {¬
|   1012     { ngx_string("connect"),            ngx_rtmp_cmd_connect_init           },¬
|   1013     { ngx_string("createStream"),       ngx_rtmp_cmd_create_stream_init     },¬
|   1014     { ngx_string("closeStream"),        ngx_rtmp_cmd_close_stream_init      },¬
|   1015     { ngx_string("deleteStream"),       ngx_rtmp_cmd_delete_stream_init     },¬
|   1016     { ngx_string("publish"),            ngx_rtmp_cmd_publish_init           },¬
|   1017     { ngx_string("play"),               ngx_rtmp_cmd_play_init              },¬
|   1018     { ngx_string("play2"),              ngx_rtmp_cmd_play2_init             },¬
|   1019     { ngx_string("seek"),               ngx_rtmp_cmd_seek_init              },¬
|   1020     { ngx_string("pause"),              ngx_rtmp_cmd_pause_init             },¬
|   1021     { ngx_string("pauseraw"),           ngx_rtmp_cmd_pause_init             },¬
|   1022 };¬

ngx_rtmp_cmd_module.c

 82 ngx_module_t  ngx_rtmp_cmd_module = {¬
|     83     NGX_MODULE_V1,¬
|     84     &ngx_rtmp_cmd_module_ctx,               /* module context */¬
|     85     NULL,                                   /* module directives */¬
|     86     NGX_RTMP_MODULE,                        /* module type */¬
|     87     NULL,                                   /* init master */¬
|     88     NULL,                                   /* init module */¬
|     89     NULL,                                   /* init process */¬
|     90     NULL,                                   /* init thread */¬
|     91     NULL,                                   /* exit thread */¬
|     92     NULL,                                   /* exit process */¬
|     93     NULL,                                   /* exit master */¬
|     94     NGX_MODULE_V1_PADDING¬
|     95 };¬

ngx_rtmp.h 定义

   662 extern ngx_module_t  ngx_rtmp_module;¬
664 #define NGX_RTMP_MODULE                 0x504D5452     /* "RTMP" */¬
    70 static ngx_rtmp_module_t  ngx_rtmp_cmd_module_ctx = {¬
|     71     NULL,                                   /* preconfiguration */¬
|     72     ngx_rtmp_cmd_postconfiguration,         /* postconfiguration */¬
|     73     NULL,                                   /* create main configuration */¬
|     74     NULL,                                   /* init main configuration */¬
|     75     NULL,                                   /* create server configuration */¬
|     76     NULL,                                   /* merge server configuration */¬
|     77     NULL,                                   /* create app configuration */¬
|     78     NULL                                    /* merge app configuration */¬
|     79 };¬

ngx_rtmp_cmd_postconfiguration

会将上面的connect-> connect_init数据写入到cmcf->amf

ngx_rtmp_cmd_connect_init

vim

查找tag

ctrlp 当退格时,太慢了,于是想看看unite是否有解决方案,结果unite不维护了,用denite,这厮需要vim8+python3,于是编python3

python3

centos的自带python3不好使,还是自己编吧,看readme.rst即可

编vim

./configure --enable-rubyinterp --enable-cscope --enable-multibyte --enable-luainterp --enable-python3interp --with-python3-config-dir=/usr/local/lib/python3.6/config-3.6m-x86_64-linux-gnu --enable-fail-if-missing 开始python是用的2.6的,我给他指向3.6了。老源码编译时,始终找不到新的python,我clone了一份最新的,configure就找到。感觉是cached的事,具体原因不明。

最后的选择:fzf + denite + ctrlp + neomru

最最后的选择:fzf + fzf.vim

beego

package beego
init()
  NewApp
    NewControllerRegister
      beecontext.NewContext()



BeeApp.Run()
  app.Server.Handler = app.Handlers
  app.Server.ListenAndServe()
   srv.Serve
    l.Accept
      go c.serve(ctx)
        serverHandler{c.server}.ServeHTTP(w, w.req)
  
  
然后有请求会回调ServeHTTP

func (p *ControllerRegister) ServeHTTP(rw http.ResponseWriter, r *http.Request)

log里的ip哪里来

func (p *ControllerRegister) ServeHTTP(rw http.ResponseWriter, r *http.Request)
回调ServeHTTP里面的r有RemoteAddr即ip信息
context.Input.IP()
input.Context.Request.RemoteAddr
Request        *http.Request

想用beego打印accept的ip?不行:具体见 https://github.com/astaxie/beego/issues/2550 目前解决方案

前面挂一个nginx

修改golang源代码或interface强制转换

继续。。。。

ListenAndServe
    srv.Serve
       l.Accept = tcpKeepAliveListener.Acceptj
       AcceptTCP (tcpsock.go)

最终查到,是调用了系统调用accept4…..没用函数..所以ttm不行。

https://lwn.net/Articles/304767/

python

debug不输出

print在程序后台后,就不输出了,需要用logging.warn输出

安装python2.7

wget http://python.org/ftp/python/2.7.6/Python-2.7.6.tar.xz --no-check-certificate
tar xf Python-2.7.6.tar.xz
cd Python-2.7.6
./configure --prefix=/usr/local --enable-unicode=ucs4 --enable-shared LDFLAGS="-Wl,-rpath /usr/local/lib"
make 
sudo make altinstall
sudo su
/use/local/bin/python2.7 get-pip.py
exit
pip2.7 install xxxx

SimplexxxxxxServer.py

在服务器建立一个http file server供文件传输

nginx

查找remote_ip源代码

src/http/ngx_http_variables.c ngx_string("remote_addr"), NULL, ngx_http_variable_remote_addr

ngx_http_request_t ->connection->addr_text

rvm

openssl 问题

rvm reinstall 2.2.5 --with-openssl-dir=$HOME/.rvm/usr --verify-downloads 1

grpc

用polygolt测试hello

echo '{"name": "lk"}' | java -jar polyglot.jar --command=call --endpoint=localhost:50051 --full_method=helloworld.Greeter/SayHelloAgain --proto_discovery_root=/home/jinlei1/ksyun/src/google.golang.org/grpc/examples/helloworld/helloworld

grpc-gateway-example:

echo '{"id": "9999999"}' | java -jar ~/polyglot.jar --command=call --endpoint=localhost:9099 --full_method=grpc.gateway.examples.examplepb.EchoService/Echo --proto_discovery_root=/home/jinlei1/ksyun/src/github.com/grpc-ecosystem/grpc-gateway/examples/examplepb --add_protoc_includes /home/jinlei1/ksyun/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis,/home/jinlei1/tools/include,${HOME}/ksyun/src/github.com/grpc-ecosystem/grpc-gateway

test the gateway:

curl localhost:8080/v1/example/echo/1 -XPOST

kafka

Consumer Id Registry

In addition to the group_id which is shared by all consumers in a group, each consumer is given a transient, unique consumer_id (of the form hostname:uuid) for identification purposes. Consumer ids are registered in the following directory.

    /consumers/[group_id]/ids/[consumer_id] --> {"version":...,"subscription":{...:...},"pattern":...,"timestamp":...} (ephemeral node)
    
Each of the consumers in the group registers under its group and creates a znode with its consumer_id. The value of the znode contains a map of <topic, #streams>. This id is simply used to identify each of the consumers which is currently active within a group. This is an ephemeral node so it will disappear if the consumer process dies.
Consumer Offsets

Consumers track the maximum offset they have consumed in each partition. This value is stored in a ZooKeeper directory if offsets.storage=zookeeper.

    /consumers/[group_id]/offsets/[topic]/[partition_id] --> offset_counter_value (persistent node)
    
Partition Owner registry

Each broker partition is consumed by a single consumer within a given consumer group. The consumer must establish its ownership of a given partition before any consumption can begin. To establish its ownership, a consumer writes its own id in an ephemeral node under the particular broker partition it is claiming.

    /consumers/[group_id]/owners/[topic]/[partition_id] --> consumer_node_id (ephemeral node)