Evaluation Benchmark on the trained model

Hi, i am trying to reproduce the papers results. I re-trained the Keras model for 500 epochs (as written in the paper) to replicate the originally returned code snippets. I followed the readme instructions and still couldn’t reach the paper’s results. I then tried updating the reload param in configs.py to be 500 and got the following responses for queries from the benchmark:

For the query: iterate through a hashmap I got the responses:

('void startBlock ( final int buildingCount ) { SwingUtilities . invokeLater ( new Runnable ( ) { @ Override public void run ( ) { statusLabel . setText ( "Computing-blockades" ) ; if ( buildingCount == 0 ) { blockadeProgress . setMaximum ( 1 ) ; blockadeProgress . setValue ( 1 ) ; } else { blockadeProgress . setMaximum ( buildingCount ) ; blockadeProgress . setValue ( 0 ) ; block = 0 ; } } } ) ; } \n', 0.035571914)

('public static void main ( String [ ] args ) { long maxMemory = runtime . maxMemory ( ) ; long allocatedMemory = runtime . totalMemory ( ) ; long freeMemory = runtime . freeMemory ( ) ; System . out . println ( "free-memory:-" + freeMemory / 1024 ) ; System . out . println ( "allocated-memory:-" + allocatedMemory / 1024 ) ; System . out . println ( "max-memory:-" + maxMemory / 1024 ) ; System . out . println ( "total-free-memory:-" + ( freeMemory + ( maxMemory - allocatedMemory ) ) / 1024 ) ; } \n', 0.033702463)

('@ Override public boolean equals ( Object obj ) { if ( ! ( obj instanceof ByteOrderMark ) ) { return false ; } ByteOrderMark bom = ( ByteOrderMark ) obj ; if ( bytes . length != bom . length ( ) ) { return false ; } for ( int i = 0 ; i < bytes . length ; i ++ ) { if ( bytes [ i ] != bom . get ( i ) ) { return false ; } } return true ; } \n', 0.03228843)

('static public RtspSession create ( String sessionId ) { if ( sessions . get ( sessionId ) != null ) { log . error ( "Session-key-conflit!!" ) ; return null ; } RtspSession session = new RtspSession ( sessionId ) ; sessions . put ( sessionId , session ) ; log . debug ( "New-session-created---id=" + sessionId ) ; return session ; } \n', 0.03165227)

('public List < Resource > getResources ( ) { if ( resources == null ) { resources = new ArrayList < Resource > ( ) ; } return this . resources ; } \n', 0.031491578)

('public List < String > getSingleCommentDelimiters ( ) { return new ArrayList < String > ( singleCommentDelimiters ) ; } \n', 0.031074077)

('public String getCanonicalHostName ( ) { try { return getHostByAddrImpl ( this ) . hostName ; } catch ( UnknownHostException ex ) { return getHostAddress ( ) ; } } \n', 0.030964658)

('public static < K , V > Map < K , V > createNewMap ( ) { return new HashMap < K , V > ( ) ; } \n', 0.030577734)

('public static String separatorsToWindows ( String path ) { if ( path == null || path . indexOf ( UNIX_SEPARATOR ) == - 1 ) { return path ; } return path . replace ( UNIX_SEPARATOR , WINDOWS_SEPARATOR ) ; } \n', 0.030175861)

('public synchronized void shutdown ( ) { dataQueue . clear ( ) ; shutdown = true ; } \n', 0.030141229)

('@ Override public void run ( ) { while ( ! destroyed ) { StreamOutput processStream = StreamOutput . EMPTY ; try { CapturedStreamOutput capturedProcessStream = null ; while ( ! destroyed && capturedProcessStream == null ) { synchronized ( toCapture ) { if ( toCapture . containsKey ( key ) ) { capturedProcessStream = toCapture . remove ( key ) ; } else { toCapture . wait ( ) ; } } } if ( ! destroyed ) { processStream = capturedProcessStream ; capturedProcessStream . readAndClose ( ) ; } } catch ( InterruptedException e ) { logger . info ( "OutputCapture-interrupted,-exiting" ) ; break ; } catch ( IOException e ) { logger . error ( "Error-reading-process-output" , e ) ; } finally { synchronized ( fromCapture ) { fromCapture . put ( key , processStream ) ; fromCapture . notify ( ) ; } } } } \n', 0.029915098)

('public static Document parse ( File file ) throws IOException , ParserConfigurationException , SAXException { DocumentBuilderFactory factory = DocumentBuilderFactory . newInstance ( ) ; DocumentBuilder builder = factory . newDocumentBuilder ( ) ; return builder . parse ( file ) ; } \n', 0.02982778)

('private static String _latexFromHtml ( Collection col , String latex ) { latex = latex . replaceAll ( "<br(-/)?>|<div>" , "-" ) ; latex = Utils . stripHTML ( latex ) ; return latex ; } \n', 0.029724797)

('public synchronized boolean clearExpired ( final Date date ) { if ( date == null ) { return false ; } boolean removed = false ; for ( Iterator < Cookie > it = cookies . iterator ( ) ; it . hasNext ( ) ; ) { if ( it . next ( ) . isExpired ( date ) ) { it . remove ( ) ; removed = true ; } } return removed ; } \n', 0.029710343)

('private String getAssetSource ( String assetPath ) { if ( StringUtils . isNotBlank ( assetPath ) ) { File asset = new File ( rootDir + File . separator + assetPath ) ; if ( asset . exists ( ) && asset . canRead ( ) ) { try { return FileUtils . readFileToString ( asset ) ; } catch ( IOException e ) { LOG . error ( "Error-reading-asset-source:-" + asset . getPath ( ) , e ) ; } } } return null ; } \n', 0.02968416)

Where only one was related to hash map (and no iteration)

for the query: convert an inputstream to a string I got the responses:

('public static void registerTarget ( javax . rmi . CORBA . Tie tie , java . rmi . Remote target ) { if ( utilDelegate != null ) { utilDelegate . registerTarget ( tie , target ) ; } } \n', 0.059240054)

('public Quaternion normalise ( Quaternion dest ) { return normalise ( this , dest ) ; } \n', 0.05841601)

('public static IntFunction chain ( final IntFunction g , final IntFunction h ) { return new IntFunction ( ) { public final int apply ( int a ) { return g . apply ( h . apply ( a ) ) ; } } ; } \n', 0.057828866)

('void focusWheel ( ) { setFocusType ( 1 ) ; } \n', 0.05775229)

('public ServerSocket createServerSocket ( int port ) throws IOException { _logger . debug ( "PortalSocketFactory.createServerPort()---port=" + port ) ; ServerSocket srvsocket = new ServerSocket ( port ) ; _logger . debug ( "PortalSocketFactory-returned-new-ServerSocket(port)..." ) ; return ( srvsocket ) ; } \n', 0.05753772)

('@ Override public final long evaluateAdj ( final int [ ] adj ) { int a , old_i , cur_i ; long totalDist ; final double [ ] m ; double phi1 , phi2 , chi1 , chi2 , twoth1 , twoth2 , t ; old_i = 1 ; totalDist = 0l ; m = this . m_coords ; a = ( ( old_i - 1 ) * 3 ) ; phi2 = m [ a ++ ] ; chi2 = m [ a ++ ] ; twoth2 = m [ a ++ ] ; for ( ; ; ) { cur_i = adj [ old_i - 1 ] ; a = ( ( cur_i - 1 ) * 3 ) ; phi1 = m [ a ++ ] ; chi1 = m [ a ++ ] ; twoth1 = m [ a ++ ] ; t = Math . abs ( phi1 - phi2 ) ; totalDist += ( ( int ) ( 0.5d + ( Math . max ( Math . min ( t , Math . abs ( t - ( 360d ) ) ) , Math . max ( Math . abs ( chi1 - chi2 ) , Math . abs ( twoth1 - twoth2 ) ) ) ) ) ) ; phi2 = phi1 ; chi2 = chi1 ; twoth2 = twoth1 ; if ( cur_i == 1 ) { return totalDist ; } old_i = cur_i ; } } \n', 0.05679653)

('public boolean equals ( Object o ) { return m_Root . equals ( ( ( Trie ) o ) . getRoot ( ) ) ; } \n', 0.056255654)

('@ Override public P2pActivityCorrections fetchByPrimaryKey ( Serializable primaryKey ) throws SystemException { return fetchByPrimaryKey ( ( ( Long ) primaryKey ) . longValue ( ) ) ; } \n', 0.056060914)

('public int getWidth ( ) { init ( ) ; return width ; } \n', 0.056045968)

('public void dispose ( ) { logDebug ( "Disposing." ) ; mSetupDone = false ; if ( mServiceConn != null ) { logDebug ( "Unbinding-from-service." ) ; if ( mContext != null ) mContext . unbindService ( mServiceConn ) ; } mDisposed = true ; mContext = null ; mServiceConn = null ; mService = null ; mPurchaseListener = null ; } \n', 0.055970065)

('private List < PrivacyListItem > tranformPrivacyItemsToPrivacyListItems ( List < PrivacyItem > items ) { List < PrivacyListItem > rItems = new ArrayList < PrivacyListItem > ( ) ; for ( int i = 0 ; i < items . size ( ) ; i ++ ) { rItems . add ( new PrivacyListItem ( items . get ( i ) . getType ( ) . ordinal ( ) , items . get ( i ) . getValue ( ) ) ) ; } return rItems ; } \n', 0.055890348)

('public BigInteger multiply ( BigInteger val ) { if ( val . signum == 0 || signum == 0 ) return ZERO ; int [ ] result = multiplyToLen ( mag , mag . length , val . mag , val . mag . length , null ) ; result = trustedStripLeadingZeroInts ( result ) ; return new BigInteger ( result , signum == val . signum ? 1 : - 1 ) ; } \n', 0.05568312)

('public java . util . Iterator getAllMembers ( ) throws GroupsException { return primGetAllMembers ( new HashSet ( ) ) . iterator ( ) ; } \n', 0.055669665)

('public void checkTextOnly ( ) { for ( int i = 0 ; i < size ; i ++ ) { ( ( CharSequence ) parts [ i ] ) . getClass ( ) ; } } \n', 0.055539)

('public void drawBarsOnGraph ( Graphics2D g2d , ArrayList < ComparableLabel > orderedDateSet , HashMap < ComparableLabel , Integer [ ] > barDataPoints , long yMaxMark ) { int sectionWidth = this . graphWidth / orderedDateSet . size ( ) ; int xOffset = sectionWidth / 2 ; int yValue ; float yOffsetPerc ; int numberOfBars = barDataPoints . get ( orderedDateSet . get ( 0 ) ) . length ; int barWidth = sectionWidth / ( numberOfBars + 1 ) ; for ( int datePos = 0 ; datePos < orderedDateSet . size ( ) ; datePos ++ ) { for ( int barNumber = 0 ; barNumber < numberOfBars ; barNumber ++ ) { yValue = barDataPoints . get ( orderedDateSet . get ( datePos ) ) [ barNumber ] ; yOffsetPerc = yValue / ( float ) yMaxMark ; int xLeftBar = this . graphLeft + datePos * sectionWidth + barWidth / 2 + barWidth * barNumber ; drawBar ( g2d , Math . round ( this . graphHeight * yOffsetPerc ) , barWidth , xLeftBar , this . graphBottom , this . barColors [ barNumber ] ) ; if ( Math . round ( this . graphHeight * yOffsetPerc ) == 0 && yValue != 0 ) { g2d . setColor ( this . barColors [ barNumber ] ) ; g2d . drawLine ( xLeftBar , this . graphBottom , xLeftBar + barWidth , this . graphBottom ) ; } } } } \n', 0.0555174)

And similiar quality results which don’t fit the result table for the other queries.

[matanpugach]

@guxd i re-trained the model. Here are the results i now got for convert an inputstream to a string

('public static String chomp ( final String str , final String separator ) { if ( StringUtils . isEmpty ( str ) || separator == null ) { return str ; } if ( str . endsWith ( separator ) ) { return str . substring ( 0 , str . length ( ) - separator . length ( ) ) ; } return str ; } \n', 0.3940239)

('protected void doStop ( ServiceStopper stopper ) throws Exception { super . doStop ( stopper ) ; if ( socket != null ) { try { socket . leaveGroup ( getMulticastAddress ( ) ) ; } catch ( IOException e ) { stopper . onException ( this , e ) ; } socket . close ( ) ; } } \n', 0.3898816)

('private JpaFinderProxy . ReturnType determineReturnType ( Class < ? > returnClass ) { if ( Collection . class . isAssignableFrom ( returnClass ) ) { return JpaFinderProxy . ReturnType . COLLECTION ; } else if ( returnClass . isArray ( ) ) { return JpaFinderProxy . ReturnType . ARRAY ; } return JpaFinderProxy . ReturnType . PLAIN ; } \n', 0.3898816)

('void addLink ( URI link ) { if ( link == null ) { throw new IllegalArgumentException ( "Missing-Link" ) ; } List < URI > copy = new ArrayList < URI > ( links . size ( ) + 1 ) ; copy . addAll ( links ) ; copy . add ( link ) ; links = Collections . unmodifiableList ( copy ) ; } \n', 0.3831619)

('public static short [ ] subarray ( final short [ ] array , int startIndexInclusive , int endIndexExclusive ) { if ( array == null ) { return null ; } if ( startIndexInclusive < 0 ) { startIndexInclusive = 0 ; } if ( endIndexExclusive > array . length ) { endIndexExclusive = array . length ; } int newSize = endIndexExclusive - startIndexInclusive ; if ( newSize <= 0 ) { return ArrayUtils . EMPTY_SHORT_ARRAY ; } short [ ] subarray = new short [ newSize ] ; System . arraycopy ( array , startIndexInclusive , subarray , 0 , newSize ) ; return subarray ; } \n', 0.3831619)

('private DateFormat getFormat ( final String pattern ) { DateFormat format = new SimpleDateFormat ( pattern ) ; if ( this . timeZone != null ) { format . setTimeZone ( this . timeZone ) ; } return format ; } \n', 0.3831619)

('public void clear ( ) { history . clear ( ) ; historyPointer = - 1 ; } \n', 0.3826081)

('private void getCoordinates ( View v , int coords [ ] ) { int p [ ] = new int [ 2 ] ; v . getLocationInWindow ( p ) ; coords [ 0 ] = p [ 0 ] ; coords [ 1 ] = p [ 1 ] ; getLocationInWindow ( p ) ; coords [ 0 ] -= p [ 0 ] ; coords [ 1 ] -= p [ 1 ] ; coords [ 2 ] = coords [ 0 ] + v . getWidth ( ) ; coords [ 3 ] = coords [ 1 ] + v . getHeight ( ) ; } \n', 0.3826081)

('public void setSelection ( GridItem [ ] _items ) { checkWidget ( ) ; if ( ! selectionEnabled ) return ; if ( _items == null ) SWT . error ( SWT . ERROR_NULL_ARGUMENT ) ; if ( selectionType == SWT . SINGLE && _items . length > 1 ) return ; if ( ! cellSelectionEnabled ) { selectedItems . clear ( ) ; } else { selectedCells . clear ( ) ; } for ( int i = 0 ; i < _items . length ; i ++ ) { GridItem item = _items [ i ] ; if ( item == null ) continue ; if ( item . isDisposed ( ) ) SWT . error ( SWT . ERROR_INVALID_ARGUMENT ) ; if ( item . getParent ( ) != this ) continue ; if ( ! cellSelectionEnabled ) { selectedItems . add ( item ) ; } else { selectCells ( getCells ( item ) ) ; } } redraw ( ) ; } \n', 0.3826081)

('public void removeCodeAttributes ( ) { code_attrs_vec . clear ( ) ; } \n', 0.3826081)

thanks for your patience & assistance !

[guxd]

@skye95git Perhaps you forgot to set the reload hyperparameter in config.py. I have added an assertion for this. Please check it out.