Calculate list of unique IP addresses

[kevin]

  Calculate list of unique IP addresses

    Just testings a few ways counting unique IP address from a pseudo random stream of what would normally be web traffic log file data , which is just some randomly filled array in the test.

     String Version

PlayBASIC Code: [Select]

   #if pbdebugmode=0
         #abort "this code needs to the compiled and run in a debugmode (f6/f7 from IDE)"
   #endif

   Dim IP$(25000)

   // fill array with IP address styled strings
   For lp =0 to getarrayelements(IP$())

         if computenewip=0

            // make 32bit IP from a couple of 16bit numbers   
            a=rnd($10000)
            b=rnd($10000)

            // stick the strings together
            s$=str$(rgbg(a))+"."+str$(rgbb(a))+"."+str$(rgbg(b))+"."+Str$(rgbb(b))
            
            // sping the wheel to see how long this run last for.
            computenewip=rnd(10)
         else

            computenewip--

         endif

         ip$(lp)=s$
   next


   // compute a list of the unique Ip addresses from this random batch
   t=timer()
   Dim Uniques$(1000)
   count=CountUniques(Uniques$())
   print "Unique IP addresses:"+str$(count)+" from "+str$(getarrayelements(IP$()))
   print timer()-t   

   #print Make$("-",128)
   #print "Unique IP Addresses"
   #print Make$("-",128)
   
   For lp =0 to count-1
         #print Uniques$(lp)
   next

   Sync
   waitkey
   

Function CountUniques(Uniques$())

   UniqueArraySize=getarrayelements(uniques$())

   Dim Heap$(255)

   LastHeapKey=-1   
   LastIP$=""
   For lp =0 to getarrayelements(IP$())
         ThisIP$=ip$(lp)
         
         if ThisIP$<>LastIP$         

            // grab the first digtit from IP string     
            Pos=instring(ThisIP$,".")

            // convert that value to integer
            Key=val(Left$(ThisIP$,pos-1))

            // check if it's the same as last, if so, cache it 
            if Key<>LastHeapKey
               KeyRow$=Heap$(Key)
               LastHeapKey=Key
            endif

            // 
            if instring(KeyRow$,ThisIP$+"~")=false

                  if count+1=>UniqueArraySize
                        UniqueArraySize=count+1000
                        redim uniques$(UniqueArraySize)                  
                  endif

                  // Add this unique address
                  uniques$(Count)=ThisIP$      
                  Count++

                  // append this address to the heap also
                  Heap$(Key)=KeyRow$+ThisIP$+"~"
            endif            
            
            LastIP$=ThisIP$   
         endif

   next

EndFunction Count


 

      Integer Array Version

PlayBASIC Code: [Select]

   #if pbdebugmode=0
         #abort "this code needs to bee compiled and run in a debug mode (f6 / f7 from IDE)"
   #endif

   Dim IP$(25000)

   // fill array with IP address styled strings
   For lp =0 to getarrayelements(IP$())

         if computenewip=0

            // make 32bit IP from a couple of 16bit numbers   
            a=rnd($10000)
            b=rnd($10000)

            // stick the strings together
            s$=str$(rgbg(a))+"."+str$(rgbb(a))+"."+str$(rgbg(b))+"."+Str$(rgbb(b))
            
            // sping the wheel to see how long this run will last for.
            computenewip=rnd(10)
         else

            computenewip--
         endif

         ip$(lp)=s$
   next


   // compute a list of the unique Ip addresses from this random batch
   t=timer()
   Dim UniquesList2$(1000)
   count=CountUniques2(UniquesList2$())
   print "Unique IP addresses:"+str$(count)+" from "+str$(getarrayelements(IP$()))
   print timer()-t   

   #print Make$("-",128)
   #print "Unique IP Addresses"
   #print Make$("-",128)
   
   For lp =0 to count-1
         #print UniquesList2$(lp)
   next

   Sync
   waitkey
   


Function CountUniques2(Uniques$())

   UniqueArraySize=getarrayelements(uniques$())

   makearray Cache()

   Dim ArrayHeap(255)

   For lp =0 to getarrayelements(IP$())

         ThisIP$=ip$(lp)
         
         if ThisIP$<>LastIP$         

            IntegerIP=val(replace$(ThisIP$,".",""))

            // convert that header value to integer
            Key=rgba(IntegerIP)
            
            // check if it's the same as last, if so, cache it 
            if ArrayHeap(key)=0
                  ArrayHeap(key)=MakeIntegerArray(32)
            endif

            Cache()=ArrayHeap(key)

            CacheArraySize=Getarrayelements(cache())

            if findarraycell(Cache(),1,1,IntegerIP,CacheArraySize) =-1
   
                  if count+1=>UniqueArraySize
                        UniqueArraySize=count+1000
                        redim uniques$(UniqueArraySize)                  
                  endif

                  // Add this unique address
                  uniques$(Count)=ThisIP$      
                  Count++

                  ItemsInCache=cache(0)+1
                  if ItemsInCache>=CacheArraySize
                     Redim Cache(CacheArraySize*2)
                  endif            

                  cache(ItemsInCache)=IntegerIP                  
                  cache(0)=ItemsInCache      
            endif            
            
            LastIP$=ThisIP$   
         endif

   next


EndFUnction Count


Function MakeIntegerArray(Size)
      Dim Stuff(size)
EndFunction Stuff()